[AMD][CI] Temporarily bypass local-registry image pulls (#36171)
This commit is contained in:
@@ -158,15 +158,7 @@ find_latest_image() {
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# If not found locally, fall back to pulling from public registry.
|
# If not found locally, resolve the latest tag from the public registry.
|
||||||
# We intentionally do not probe ${LOCAL_DOCKER_REGISTRY} here with
|
|
||||||
# `docker manifest inspect --insecure` because that command runs in the
|
|
||||||
# runner pod's network namespace, which on every observed AMD scale set
|
|
||||||
# cannot reach 10.44.14.109:5000 (every probe either fast-fails with TLS
|
|
||||||
# reject or hits a 30s TCP timeout, multiplied across 7 daily candidates).
|
|
||||||
# The actual local-registry pull still happens in the call site below via
|
|
||||||
# `docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}"`, which goes through the
|
|
||||||
# docker daemon on the host and inherits its insecure-registries config.
|
|
||||||
for days_back in {0..6}; do
|
for days_back in {0..6}; do
|
||||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||||
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
||||||
@@ -272,19 +264,9 @@ elif [[ -n "${BUILD_FROM_DOCKERFILE}" ]]; then
|
|||||||
else
|
else
|
||||||
# Find the latest pre-built image
|
# Find the latest pre-built image
|
||||||
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
||||||
# Try the local docker registry first (avoids Docker Hub rate limits and is
|
# Temporarily bypass the shared local registry while concurrent CI pulls
|
||||||
# faster on the LAN); if that fails for any reason, fall back to the
|
# saturate it. Keep using the authenticated, retried public-registry path.
|
||||||
# public registry with exponential-backoff retries. Capture stderr so the
|
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||||
# real failure reason (TLS handshake, 404, connection refused, etc.) is
|
|
||||||
# visible in the job log instead of being silently swallowed.
|
|
||||||
if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
|
|
||||||
echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
|
|
||||||
docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
|
|
||||||
else
|
|
||||||
echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
|
|
||||||
printf '%s\n' "${local_pull_output}" | sed 's/^/ [local-pull] /' >&2
|
|
||||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
CACHE_HOST=/home/runner/sglang-data
|
CACHE_HOST=/home/runner/sglang-data
|
||||||
|
|||||||
@@ -143,11 +143,7 @@ find_latest_image() {
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
# If not found locally, fall back to pulling from public registry.
|
# If not found locally, resolve the latest tag from the public registry.
|
||||||
# See amd_ci_start_container.sh for why we don't probe
|
|
||||||
# ${LOCAL_DOCKER_REGISTRY} with `docker manifest inspect --insecure` from
|
|
||||||
# the runner pod's network namespace; the actual local-registry pull
|
|
||||||
# happens at the call site below via the docker daemon on the host.
|
|
||||||
for days_back in {0..6}; do
|
for days_back in {0..6}; do
|
||||||
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
image_tag="${base_tag}-$(date -d "${days_back} days ago" +%Y%m%d)"
|
||||||
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
echo "Checking for image: rocm/sgl-dev:${image_tag}" >&2
|
||||||
@@ -225,19 +221,9 @@ if [[ -n "${CUSTOM_IMAGE}" ]]; then
|
|||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
IMAGE=$(find_latest_image "${GPU_ARCH}")
|
||||||
# Try the local docker registry first (avoids Docker Hub rate limits and is
|
# Temporarily bypass the shared local registry while concurrent CI pulls
|
||||||
# faster on the LAN); if that fails for any reason, fall back to the
|
# saturate it. Keep using the authenticated, retried public-registry path.
|
||||||
# public registry with exponential-backoff retries. Capture stderr so the
|
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||||
# real failure reason (TLS handshake, 404, connection refused, etc.) is
|
|
||||||
# visible in the job log instead of being silently swallowed.
|
|
||||||
if local_pull_output=$(docker pull "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" 2>&1); then
|
|
||||||
echo "Pulled from local docker registry: ${LOCAL_DOCKER_REGISTRY}/${IMAGE}"
|
|
||||||
docker tag "${LOCAL_DOCKER_REGISTRY}/${IMAGE}" "${IMAGE}"
|
|
||||||
else
|
|
||||||
echo "Local docker registry pull failed; falling back to public registry: ${IMAGE}" >&2
|
|
||||||
printf '%s\n' "${local_pull_output}" | sed 's/^/ [local-pull] /' >&2
|
|
||||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
|
||||||
fi
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
CACHE_HOST=/home/runner/sglang-data
|
CACHE_HOST=/home/runner/sglang-data
|
||||||
|
|||||||
Reference in New Issue
Block a user