[AMD] mirror nightly images to local registry and prefer LAN pulls (#23073)
Co-authored-by: bingxche <bingxche@amd.com>
This commit is contained in:
YC Yen-Ching Tseng
co-authored by
bingxche
bingxche
parent
8c13295842
commit
f399997d2f
@@ -82,6 +82,22 @@ jobs:
|
||||
docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg NIC_BACKEND=ainic --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
|
||||
docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
|
||||
|
||||
# Persist the tag right after rocm/sgl-dev push succeeds so the local
|
||||
# registry mirror can run even if a later step in this job (lmsys push)
|
||||
# fails. By default this step only runs when the previous step succeeded,
|
||||
# so the artifact only exists when an image actually landed on Docker Hub.
|
||||
- name: Save published image tag
|
||||
run: |
|
||||
mkdir -p image-tag
|
||||
echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
|
||||
|
||||
- name: Upload image tag artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
path: image-tag/${{ matrix.gpu_arch }}.txt
|
||||
retention-days: 1
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
@@ -93,90 +109,136 @@ jobs:
|
||||
docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
|
||||
# Temporarily disable docker cache seeding until performant storage is in place
|
||||
cache:
|
||||
if: false
|
||||
# if: always() && github.repository == 'sgl-project/sglang'
|
||||
runs-on: linux-mi300-gpu-1
|
||||
# Mirror the freshly published rocm/sgl-dev image to the in-network Docker
|
||||
# registry so AMD CI runners can pull without hitting Docker Hub rate limits.
|
||||
# The tag is read verbatim from the publish job's artifact so this job uses
|
||||
# exactly the same tag that publish pushed (only the registry prefix differs).
|
||||
# `!cancelled()` lets us still mirror successful matrix legs when other legs
|
||||
# of publish failed; legs without an artifact will fail at download and be
|
||||
# the only ones marked red.
|
||||
push_local_registry:
|
||||
if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }}
|
||||
runs-on: linux-mi300-1gpu-sglang
|
||||
environment: 'prod'
|
||||
needs: publish
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
gpu_arch: ['gfx942']
|
||||
build_type: ['all']
|
||||
gpu_arch: ['gfx942', 'gfx950']
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
- name: Download image tag artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
fetch-depth: 0 # Required for git describe to find tags
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
|
||||
- name: "Set Date"
|
||||
- name: Read image tag
|
||||
run: |
|
||||
echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
|
||||
|
||||
- name: Get version from latest tag
|
||||
id: version
|
||||
run: |
|
||||
# Get the latest version tag sorted by version number (e.g., v0.5.7 -> 0.5.7)
|
||||
VERSION=$(git tag -l 'v[0-9]*' --sort=-v:refname | head -1 | sed 's/^v//')
|
||||
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "::error::Could not determine version from git tags"
|
||||
image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
|
||||
if [ -z "${image_tag}" ]; then
|
||||
echo "::error::Image tag artifact is empty"
|
||||
exit 1
|
||||
fi
|
||||
echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
|
||||
echo "Resolved IMAGE_TAG=${image_tag}"
|
||||
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "Detected version: ${VERSION}"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
- name: Login to Docker Hub (AMD)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
- name: Pull and Save Docker Image to Cache
|
||||
- name: Mirror rocm/sgl-dev to local registry
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
|
||||
dst="172.29.8.23:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
|
||||
docker pull "${src}"
|
||||
docker tag "${src}" "${dst}"
|
||||
docker push "${dst}"
|
||||
|
||||
version=${{ steps.version.outputs.version }}
|
||||
echo "Version: ${version}"
|
||||
# Temporarily disable docker cache seeding until performant storage is in place
|
||||
# cache:
|
||||
# if: false
|
||||
# # if: always() && github.repository == 'sgl-project/sglang'
|
||||
# runs-on: linux-mi300-gpu-1
|
||||
# environment: 'prod'
|
||||
# needs: publish
|
||||
# strategy:
|
||||
# fail-fast: false
|
||||
# matrix:
|
||||
# gpu_arch: ['gfx942']
|
||||
# build_type: ['all']
|
||||
# steps:
|
||||
# - name: Checkout repository
|
||||
# uses: actions/checkout@v4
|
||||
# with:
|
||||
# fetch-depth: 0 # Required for git describe to find tags
|
||||
|
||||
if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
|
||||
rocm_tag="rocm700-mi30x"
|
||||
else
|
||||
echo "Unsupported gfx arch"
|
||||
exit 1
|
||||
fi
|
||||
# - name: "Set Date"
|
||||
# run: |
|
||||
# echo "DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
|
||||
|
||||
tag=v${version}-${rocm_tag}
|
||||
# - name: Get version from latest tag
|
||||
# id: version
|
||||
# run: |
|
||||
# # Get the latest version tag sorted by version number (e.g., v0.5.7 -> 0.5.7)
|
||||
# VERSION=$(git tag -l 'v[0-9]*' --sort=-v:refname | head -1 | sed 's/^v//')
|
||||
|
||||
if [ "${{ matrix.build_type }}" = "all" ]; then
|
||||
tag_suffix=""
|
||||
else
|
||||
echo "Unsupported build type"
|
||||
exit 1
|
||||
fi
|
||||
# if [ -z "$VERSION" ]; then
|
||||
# echo "::error::Could not determine version from git tags"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}"
|
||||
# echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
# echo "Detected version: ${VERSION}"
|
||||
|
||||
# Determine target cache file name based on ROCm variant
|
||||
if [[ "${rocm_tag}" == rocm700* ]]; then
|
||||
final_path="/home/runner/sgl-data/docker/image-700.tar"
|
||||
else
|
||||
echo "Unexpected ROCm tag: ${rocm_tag}"
|
||||
exit 1
|
||||
fi
|
||||
# - name: Login to Docker Hub
|
||||
# uses: docker/login-action@v2
|
||||
# with:
|
||||
# username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
# password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
tmp_path="${final_path}.tmp"
|
||||
# - name: Pull and Save Docker Image to Cache
|
||||
# run: |
|
||||
# set -euxo pipefail
|
||||
|
||||
echo "Pulling image: ${image}"
|
||||
docker pull "${image}"
|
||||
# version=${{ steps.version.outputs.version }}
|
||||
# echo "Version: ${version}"
|
||||
|
||||
echo "Saving to temp file: ${tmp_path}"
|
||||
docker save "${image}" -o "${tmp_path}"
|
||||
# if [ "${{ matrix.gpu_arch }}" = "gfx942" ]; then
|
||||
# rocm_tag="rocm700-mi30x"
|
||||
# else
|
||||
# echo "Unsupported gfx arch"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
echo "Moving to final path: ${final_path}"
|
||||
mv -f "${tmp_path}" "${final_path}"
|
||||
# tag=v${version}-${rocm_tag}
|
||||
|
||||
echo "Cache populated successfully at ${final_path}"
|
||||
# if [ "${{ matrix.build_type }}" = "all" ]; then
|
||||
# tag_suffix=""
|
||||
# else
|
||||
# echo "Unsupported build type"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# image="rocm/sgl-dev:${tag}-${{ env.DATE }}${tag_suffix}"
|
||||
|
||||
# # Determine target cache file name based on ROCm variant
|
||||
# if [[ "${rocm_tag}" == rocm700* ]]; then
|
||||
# final_path="/home/runner/sgl-data/docker/image-700.tar"
|
||||
# else
|
||||
# echo "Unexpected ROCm tag: ${rocm_tag}"
|
||||
# exit 1
|
||||
# fi
|
||||
|
||||
# tmp_path="${final_path}.tmp"
|
||||
|
||||
# echo "Pulling image: ${image}"
|
||||
# docker pull "${image}"
|
||||
|
||||
# echo "Saving to temp file: ${tmp_path}"
|
||||
# docker save "${image}" -o "${tmp_path}"
|
||||
|
||||
# echo "Moving to final path: ${final_path}"
|
||||
# mv -f "${tmp_path}" "${final_path}"
|
||||
|
||||
# echo "Cache populated successfully at ${final_path}"
|
||||
|
||||
@@ -82,6 +82,22 @@ jobs:
|
||||
docker build . -f docker/rocm.Dockerfile --build-arg SGL_BRANCH=${{ github.ref_name }} --build-arg BUILD_TYPE=${{ matrix.build_type }} --build-arg GPU_ARCH=${{ matrix.gpu_arch }} --build-arg ENABLE_MORI=1 --build-arg NIC_BACKEND=ainic --build-arg SETUPTOOLS_SCM_PRETEND_VERSION=${pretend_version} -t rocm/sgl-dev:${tag}-${{ env.DATE }} --no-cache
|
||||
docker push rocm/sgl-dev:${tag}-${{ env.DATE }}
|
||||
|
||||
# Persist the tag right after rocm/sgl-dev push succeeds so the local
|
||||
# registry mirror can run even if a later step in this job (lmsys push)
|
||||
# fails. By default this step only runs when the previous step succeeded,
|
||||
# so the artifact only exists when an image actually landed on Docker Hub.
|
||||
- name: Save published image tag
|
||||
run: |
|
||||
mkdir -p image-tag
|
||||
echo "${{ env.IMAGE_TAG }}" > "image-tag/${{ matrix.gpu_arch }}.txt"
|
||||
|
||||
- name: Upload image tag artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
path: image-tag/${{ matrix.gpu_arch }}.txt
|
||||
retention-days: 1
|
||||
|
||||
- name: Login to Docker Hub (lmsys)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
@@ -92,3 +108,49 @@ jobs:
|
||||
run: |
|
||||
docker tag rocm/sgl-dev:${{ env.IMAGE_TAG }} lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
docker push lmsysorg/sglang-rocm:${{ env.IMAGE_TAG }}
|
||||
|
||||
# Mirror the freshly published rocm/sgl-dev image to the in-network Docker
|
||||
# registry so AMD CI runners can pull without hitting Docker Hub rate limits.
|
||||
# The tag is read verbatim from the publish job's artifact so this job uses
|
||||
# exactly the same tag that publish pushed (only the registry prefix differs).
|
||||
# `!cancelled()` lets us still mirror successful matrix legs when other legs
|
||||
# of publish failed; legs without an artifact will fail at download and be
|
||||
# the only ones marked red.
|
||||
push_local_registry:
|
||||
if: ${{ !cancelled() && github.repository == 'sgl-project/sglang' }}
|
||||
runs-on: linux-mi300-1gpu-sglang
|
||||
environment: 'prod'
|
||||
needs: publish
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
gpu_arch: ['gfx942-rocm720', 'gfx950-rocm720']
|
||||
steps:
|
||||
- name: Download image tag artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: image-tag-${{ matrix.gpu_arch }}
|
||||
|
||||
- name: Read image tag
|
||||
run: |
|
||||
image_tag=$(tr -d '[:space:]' < "${{ matrix.gpu_arch }}.txt")
|
||||
if [ -z "${image_tag}" ]; then
|
||||
echo "::error::Image tag artifact is empty"
|
||||
exit 1
|
||||
fi
|
||||
echo "IMAGE_TAG=${image_tag}" >> $GITHUB_ENV
|
||||
echo "Resolved IMAGE_TAG=${image_tag}"
|
||||
|
||||
- name: Login to Docker Hub (AMD)
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_AMD_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_AMD_TOKEN }}
|
||||
|
||||
- name: Mirror rocm/sgl-dev to local registry
|
||||
run: |
|
||||
src="rocm/sgl-dev:${{ env.IMAGE_TAG }}"
|
||||
dst="172.29.8.23:5000/rocm/sgl-dev:${{ env.IMAGE_TAG }}"
|
||||
docker pull "${src}"
|
||||
docker tag "${src}" "${dst}"
|
||||
docker push "${dst}"
|
||||
|
||||
@@ -23,6 +23,7 @@ fi
|
||||
ROCM_VERSION="rocm700"
|
||||
DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
|
||||
DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
|
||||
LOCAL_DOCKER_REGISTRY="172.29.8.23:5000"
|
||||
|
||||
# Parse command line arguments
|
||||
MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
|
||||
@@ -96,11 +97,45 @@ else
|
||||
DEVICE_FLAG="--device /dev/dri"
|
||||
fi
|
||||
|
||||
# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
|
||||
retry_with_backoff() {
|
||||
local max_attempts=$1; shift
|
||||
local attempt=1
|
||||
local wait_secs=30
|
||||
# Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
|
||||
local jitter=$(( RANDOM % 30 ))
|
||||
while true; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
if (( attempt >= max_attempts )); then
|
||||
echo "Error: '$*' failed after ${max_attempts} attempts" >&2
|
||||
return 1
|
||||
fi
|
||||
local sleep_time=$(( wait_secs + jitter ))
|
||||
echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
|
||||
sleep "${sleep_time}"
|
||||
(( attempt++ ))
|
||||
(( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
|
||||
jitter=$(( RANDOM % 30 ))
|
||||
done
|
||||
}
|
||||
|
||||
# Authenticate to Docker Hub to avoid anonymous pull rate limits.
|
||||
# Credentials are optional; when absent we fall back to unauthenticated pulls.
|
||||
if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
|
||||
echo "Logging in to Docker Hub…"
|
||||
if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
|
||||
echo "Docker Hub login successful"
|
||||
else
|
||||
echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# Find the latest image
|
||||
find_latest_image() {
|
||||
local gpu_arch=$1
|
||||
local base_tag days_back image_tag
|
||||
local base_tag days_back image_tag image_id remote_tags
|
||||
|
||||
case "${gpu_arch}" in
|
||||
mi30x) base_tag="${MI30X_BASE_TAG}" ;;
|
||||
@@ -108,19 +143,29 @@ find_latest_image() {
|
||||
*) echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
|
||||
esac
|
||||
|
||||
# First, check local cache
|
||||
# First, check local cache on the runner.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
local local_image="rocm/sgl-dev:${image_tag}"
|
||||
image_id=$(docker images -q "${local_image}")
|
||||
image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
|
||||
if [[ -n "$image_id" ]]; then
|
||||
echo "Found cached image locally: ${local_image}" >&2
|
||||
echo "${local_image}"
|
||||
return 0
|
||||
echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
|
||||
echo "rocm/sgl-dev:${image_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# If not found locally, fall back to pulling from public registry
|
||||
# Then try the local registry.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
|
||||
echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Finally, try the public registry.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
||||
@@ -135,7 +180,7 @@ find_latest_image() {
|
||||
echo "Exact version not found. Searching remote registry for any ${ROCM_VERSION}-${gpu_arch} image…" >&2
|
||||
for days_back in {0..6}; do
|
||||
local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
|
||||
local remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1)
|
||||
remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true)
|
||||
if [[ -n "$remote_tags" ]]; then
|
||||
echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
|
||||
echo "rocm/sgl-dev:${remote_tags}"
|
||||
@@ -181,7 +226,11 @@ if [[ -n "${CUSTOM_IMAGE}" ]]; then
|
||||
# Use explicitly provided custom image
|
||||
IMAGE="${CUSTOM_IMAGE}"
|
||||
echo "Using custom image: ${IMAGE}"
|
||||
docker pull "${IMAGE}"
|
||||
if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
|
||||
docker pull "${IMAGE}"
|
||||
else
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
fi
|
||||
elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
|
||||
# Build image from Dockerfile
|
||||
if [[ -z "${GPU_ARCH_BUILD}" ]]; then
|
||||
@@ -212,10 +261,19 @@ else
|
||||
# Find the latest pre-built image
|
||||
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
||||
echo "Pulling Docker image: ${IMAGE}"
|
||||
docker pull "${IMAGE}"
|
||||
if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
|
||||
# Local registry is on-LAN; no need to retry.
|
||||
docker pull "${IMAGE}"
|
||||
docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
else
|
||||
# Public registry pulls can hit rate limits; retry with backoff.
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
fi
|
||||
fi
|
||||
|
||||
CACHE_HOST=/home/runner/sgl-data
|
||||
# CACHE_HOST=/home/runner/sgl-data
|
||||
CACHE_HOST=/home/runner/sglang-data
|
||||
if [[ -d "$CACHE_HOST" ]]; then
|
||||
CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
|
||||
else
|
||||
|
||||
@@ -23,6 +23,7 @@ fi
|
||||
ROCM_VERSION="rocm700"
|
||||
DEFAULT_MI30X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi30x"
|
||||
DEFAULT_MI35X_BASE_TAG="${SGLANG_VERSION}-${ROCM_VERSION}-mi35x"
|
||||
LOCAL_DOCKER_REGISTRY="172.29.8.23:5000"
|
||||
|
||||
# Parse command line arguments
|
||||
MI30X_BASE_TAG="${DEFAULT_MI30X_BASE_TAG}"
|
||||
@@ -83,11 +84,45 @@ else
|
||||
DEVICE_FLAG="--device /dev/dri"
|
||||
fi
|
||||
|
||||
# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
|
||||
retry_with_backoff() {
|
||||
local max_attempts=$1; shift
|
||||
local attempt=1
|
||||
local wait_secs=30
|
||||
# Add jitter (0-30s) so concurrent jobs don't all retry at the same instant
|
||||
local jitter=$(( RANDOM % 30 ))
|
||||
while true; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
if (( attempt >= max_attempts )); then
|
||||
echo "Error: '$*' failed after ${max_attempts} attempts" >&2
|
||||
return 1
|
||||
fi
|
||||
local sleep_time=$(( wait_secs + jitter ))
|
||||
echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s…" >&2
|
||||
sleep "${sleep_time}"
|
||||
(( attempt++ ))
|
||||
(( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
|
||||
jitter=$(( RANDOM % 30 ))
|
||||
done
|
||||
}
|
||||
|
||||
# Authenticate to Docker Hub to avoid anonymous pull rate limits.
|
||||
# Credentials are optional; when absent we fall back to unauthenticated pulls.
|
||||
if [[ -n "${DOCKERHUB_AMD_USERNAME:-}" && -n "${DOCKERHUB_AMD_TOKEN:-}" ]]; then
|
||||
echo "Logging in to Docker Hub…"
|
||||
if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_AMD_TOKEN}" | docker login -u "${DOCKERHUB_AMD_USERNAME}" --password-stdin >/dev/null 2>&1'; then
|
||||
echo "Docker Hub login successful"
|
||||
else
|
||||
echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# Find the latest image
|
||||
find_latest_image() {
|
||||
local gpu_arch=$1
|
||||
local base_tag days_back image_tag
|
||||
local base_tag days_back image_tag image_id remote_tags
|
||||
|
||||
case "${gpu_arch}" in
|
||||
mi30x) base_tag="${MI30X_BASE_TAG}" ;;
|
||||
@@ -95,19 +130,29 @@ find_latest_image() {
|
||||
*) echo "Error: unsupported GPU architecture '${gpu_arch}'" >&2; return 1 ;;
|
||||
esac
|
||||
|
||||
# First, check local cache
|
||||
# First, check local cache on the runner.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
local local_image="rocm/sgl-dev:${image_tag}"
|
||||
image_id=$(docker images -q "${local_image}")
|
||||
image_id=$(docker images -q "rocm/sgl-dev:${image_tag}")
|
||||
if [[ -n "$image_id" ]]; then
|
||||
echo "Found cached image locally: ${local_image}" >&2
|
||||
echo "${local_image}"
|
||||
return 0
|
||||
echo "Found cached image locally: rocm/sgl-dev:${image_tag}" >&2
|
||||
echo "rocm/sgl-dev:${image_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# If not found locally, fall back to pulling from public registry
|
||||
# Then try the local registry.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
if docker manifest inspect --insecure "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >/dev/null 2>&1; then
|
||||
echo "Found available image: ${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}" >&2
|
||||
echo "${LOCAL_DOCKER_REGISTRY}/rocm/sgl-dev:${image_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Finally, try the public registry.
|
||||
for days_back in {0..6}; do
|
||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
||||
@@ -122,7 +167,7 @@ find_latest_image() {
|
||||
echo "Exact version not found. Searching remote registry for any ${ROCM_VERSION}-${gpu_arch} image…" >&2
|
||||
for days_back in {0..6}; do
|
||||
local target_date=$(date -d "${days_back} days ago" +%Y%m%d)
|
||||
local remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1)
|
||||
remote_tags=$(curl -s "https://registry.hub.docker.com/v2/repositories/rocm/sgl-dev/tags?page_size=100&name=${ROCM_VERSION}-${gpu_arch}-${target_date}" 2>/dev/null | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true)
|
||||
if [[ -n "$remote_tags" ]]; then
|
||||
echo "Found available image: rocm/sgl-dev:${remote_tags}" >&2
|
||||
echo "rocm/sgl-dev:${remote_tags}"
|
||||
@@ -166,9 +211,18 @@ find_latest_image() {
|
||||
# Pull and run the latest image
|
||||
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
||||
echo "Pulling Docker image: ${IMAGE}"
|
||||
docker pull "${IMAGE}"
|
||||
if [[ "${IMAGE}" == "${LOCAL_DOCKER_REGISTRY}/"* ]]; then
|
||||
# Local registry is on-LAN; no need to retry.
|
||||
docker pull "${IMAGE}"
|
||||
docker tag "${IMAGE}" "${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
IMAGE="${IMAGE#${LOCAL_DOCKER_REGISTRY}/}"
|
||||
else
|
||||
# Public registry pulls can hit rate limits; retry with backoff.
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
fi
|
||||
|
||||
CACHE_HOST=/home/runner/sgl-data
|
||||
# CACHE_HOST=/home/runner/sgl-data
|
||||
CACHE_HOST=/home/runner/sglang-data
|
||||
if [[ -d "$CACHE_HOST" ]]; then
|
||||
CACHE_VOLUME="-v $CACHE_HOST:/sgl-data"
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user