[XPU CI] Pull prebuilt nightly image instead of building per-stage (#27282)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
ashwini rathi
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
1c73ff8ad3
commit
acf65c1c68
@@ -73,57 +73,52 @@ jobs:
|
||||
needs: [check-changes, pr-gate]
|
||||
if: needs.check-changes.outputs.main_package == 'true'
|
||||
runs-on: intel-bmg
|
||||
env:
|
||||
DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
|
||||
DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
|
||||
steps:
|
||||
- name: Reset workspace ownership
|
||||
run: |
|
||||
# Prior runs install sglang as root inside the CI container against
|
||||
# the bind-mounted workspace, leaving root-owned egg-info on the host.
|
||||
# actions/checkout's pre-clean step can't unlink those as the runner
|
||||
# user, so reset ownership before checkout runs.
|
||||
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
|
||||
chown -R "$(id -u):$(id -g)" /w || true
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.ref || github.ref }}
|
||||
|
||||
- name: Build Docker image
|
||||
- name: Start CI container
|
||||
run: |
|
||||
PR_REPO=${{ github.event.pull_request.head.repo.clone_url }}
|
||||
PR_HEAD_REF=${{ github.head_ref }}
|
||||
docker build \
|
||||
${PR_REPO:+--build-arg SG_LANG_REPO=$PR_REPO} \
|
||||
${PR_HEAD_REF:+--build-arg SG_LANG_BRANCH=$PR_HEAD_REF} \
|
||||
--no-cache --progress=plain -f docker/xpu.Dockerfile -t xpu_sglang_main:bmg .
|
||||
|
||||
- name: Run container
|
||||
id: start_container
|
||||
run: |
|
||||
container_id=$(docker run -dt \
|
||||
--group-add 992 \
|
||||
--group-add $(getent group video | cut -d: -f3) \
|
||||
--group-add $(getent group render | cut -d: -f3) \
|
||||
-v $HOME/.cache/huggingface:/root/.cache/huggingface \
|
||||
--device /dev/dri \
|
||||
-v /dev/dri/by-path:/dev/dri/by-path \
|
||||
-e HF_TOKEN="$(cat ~/huggingface_token.txt)" \
|
||||
xpu_sglang_main:bmg)
|
||||
echo "Started container: $container_id"
|
||||
echo "container_id=$container_id" >> "$GITHUB_OUTPUT"
|
||||
export HF_TOKEN="$(cat ~/huggingface_token.txt)"
|
||||
bash scripts/ci/xpu/xpu_ci_start_container.sh
|
||||
env:
|
||||
GITHUB_WORKSPACE: ${{ github.workspace }}
|
||||
|
||||
- name: Install Dependency
|
||||
timeout-minutes: 20
|
||||
timeout-minutes: 60
|
||||
run: |
|
||||
cid="${{ steps.start_container.outputs.container_id }}"
|
||||
docker exec "$cid" /opt/venv/bin/python3 -m pip install --upgrade pip
|
||||
docker exec "$cid" /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate
|
||||
docker exec "$cid" /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python
|
||||
docker exec "$cid" /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN} '
|
||||
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --upgrade pip
|
||||
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate
|
||||
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang
|
||||
docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml
|
||||
docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu
|
||||
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
|
||||
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install triton-xpu==3.7.1 --index-url https://download.pytorch.org/whl/test/xpu --force-reinstall
|
||||
docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}'
|
||||
|
||||
- name: Run stage-a tests
|
||||
timeout-minutes: 30
|
||||
run: |
|
||||
cid="${{ steps.start_container.outputs.container_id }}"
|
||||
docker exec "$cid" bash -c "source /opt/venv/bin/activate && cd /sgl-workspace/sglang/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu"
|
||||
docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu"
|
||||
|
||||
- name: Cleanup container
|
||||
if: always()
|
||||
run: |
|
||||
cid="${{ steps.start_container.outputs.container_id }}"
|
||||
docker rm -f "$cid" || true
|
||||
run: docker rm -f ci_sglang_xpu || true
|
||||
|
||||
# ==================== Wait for Stage A ==================== #
|
||||
wait-for-stage-a:
|
||||
|
||||
Executable
+159
@@ -0,0 +1,159 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Start the Intel XPU CI container (ci_sglang_xpu) using the latest nightly
|
||||
# intel/sglang-dev image published by .github/workflows/release-docker-intel-xpu-nightly.yml.
|
||||
#
|
||||
# Walks back N days through nightly date-stamped tags, pulls the first match,
|
||||
# then starts a long-running container that subsequent steps `docker exec` into.
|
||||
|
||||
CONTAINER_NAME="ci_sglang_xpu"
|
||||
IMAGE_REPO="intel/sglang-dev"
|
||||
TAG_PREFIX="nightly-dev-xpu-bmg"
|
||||
LOOKBACK_DAYS=7
|
||||
CUSTOM_IMAGE=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--custom-image) CUSTOM_IMAGE="$2"; shift 2;;
|
||||
--container-name) CONTAINER_NAME="$2"; shift 2;;
|
||||
--lookback-days) LOOKBACK_DAYS="$2"; shift 2;;
|
||||
-h|--help)
|
||||
echo "Usage: $0 [OPTIONS]"
|
||||
echo "Options:"
|
||||
echo " --custom-image IMAGE Use a specific Docker image directly"
|
||||
echo " --container-name NAME Override container name (default: ${CONTAINER_NAME})"
|
||||
echo " --lookback-days N Days of nightly tags to scan (default: ${LOOKBACK_DAYS})"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Unknown option $1"; exit 1;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Retry a command with exponential backoff. Usage: retry_with_backoff <max_attempts> <cmd...>
|
||||
retry_with_backoff() {
|
||||
local max_attempts=$1; shift
|
||||
local attempt=1
|
||||
local wait_secs=30
|
||||
local jitter=$(( RANDOM % 30 ))
|
||||
while true; do
|
||||
if "$@"; then
|
||||
return 0
|
||||
fi
|
||||
if (( attempt >= max_attempts )); then
|
||||
echo "Error: '$*' failed after ${max_attempts} attempts" >&2
|
||||
return 1
|
||||
fi
|
||||
local sleep_time=$(( wait_secs + jitter ))
|
||||
echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s..." >&2
|
||||
sleep "${sleep_time}"
|
||||
(( attempt++ ))
|
||||
(( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 ))
|
||||
jitter=$(( RANDOM % 30 ))
|
||||
done
|
||||
}
|
||||
|
||||
# Authenticate to Docker Hub when credentials are present (avoids anonymous pull
|
||||
# rate limits). Both vars are optional; falls back to unauthenticated pulls.
|
||||
if [[ -n "${DOCKERHUB_INTEL_USERNAME:-}" && -n "${DOCKERHUB_INTEL_TOKEN:-}" ]]; then
|
||||
echo "Logging in to Docker Hub..."
|
||||
if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_INTEL_TOKEN}" | docker login -u "${DOCKERHUB_INTEL_USERNAME}" --password-stdin >/dev/null 2>&1'; then
|
||||
echo "Docker Hub login successful"
|
||||
else
|
||||
echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
# Locate the latest published nightly image. Walks back LOOKBACK_DAYS days of
|
||||
# date-stamped tags (commit-hash suffix is unknown, so we query Docker Hub).
|
||||
find_latest_image() {
|
||||
local days_back target_date matched_tag
|
||||
|
||||
# Local cache first.
|
||||
for (( days_back=0; days_back<LOOKBACK_DAYS; days_back++ )); do
|
||||
target_date=$(date -d "${days_back} days ago" +%Y%m%d)
|
||||
matched_tag=$(docker images --format '{{.Repository}}:{{.Tag}}' \
|
||||
--filter "reference=${IMAGE_REPO}:${TAG_PREFIX}-${target_date}-*" \
|
||||
| sort -r | head -n 1)
|
||||
if [[ -n "${matched_tag}" ]]; then
|
||||
echo "Found cached image locally: ${matched_tag}" >&2
|
||||
echo "${matched_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Then query Docker Hub for any tag matching the date prefix.
|
||||
for (( days_back=0; days_back<LOOKBACK_DAYS; days_back++ )); do
|
||||
target_date=$(date -d "${days_back} days ago" +%Y%m%d)
|
||||
echo "Checking Docker Hub for: ${IMAGE_REPO}:${TAG_PREFIX}-${target_date}-*" >&2
|
||||
matched_tag=$(curl -fsSL \
|
||||
"https://registry.hub.docker.com/v2/repositories/${IMAGE_REPO}/tags?page_size=100&name=${TAG_PREFIX}-${target_date}" \
|
||||
2>/dev/null \
|
||||
| grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true)
|
||||
if [[ -n "${matched_tag}" ]]; then
|
||||
echo "Found published image: ${IMAGE_REPO}:${matched_tag}" >&2
|
||||
echo "${IMAGE_REPO}:${matched_tag}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
# Final fallback: any locally cached image matching the prefix.
|
||||
matched_tag=$(docker images --format '{{.Repository}}:{{.Tag}}' \
|
||||
--filter "reference=${IMAGE_REPO}:${TAG_PREFIX}-*" \
|
||||
| sort -r | head -n 1)
|
||||
if [[ -n "${matched_tag}" ]]; then
|
||||
echo "Using cached fallback image: ${matched_tag}" >&2
|
||||
echo "${matched_tag}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Error: no ${IMAGE_REPO}:${TAG_PREFIX}-* image found in the last ${LOOKBACK_DAYS} days" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ -n "${CUSTOM_IMAGE}" ]]; then
|
||||
IMAGE="${CUSTOM_IMAGE}"
|
||||
echo "Using custom image: ${IMAGE}"
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
else
|
||||
IMAGE=$(find_latest_image)
|
||||
if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then
|
||||
retry_with_backoff 6 docker pull "${IMAGE}"
|
||||
else
|
||||
echo "Image ${IMAGE} already present locally; skipping pull"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Remove any stale container of the same name so re-runs are idempotent.
|
||||
if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then
|
||||
echo "Removing existing container: ${CONTAINER_NAME}"
|
||||
docker rm -f "${CONTAINER_NAME}" >/dev/null
|
||||
fi
|
||||
|
||||
VIDEO_GID=$(getent group video | cut -d: -f3)
|
||||
RENDER_GID=$(getent group render | cut -d: -f3)
|
||||
|
||||
HF_TOKEN_FILE="${HOME}/huggingface_token.txt"
|
||||
HF_TOKEN_VALUE=""
|
||||
if [[ -n "${HF_TOKEN:-}" ]]; then
|
||||
HF_TOKEN_VALUE="${HF_TOKEN}"
|
||||
elif [[ -r "${HF_TOKEN_FILE}" ]]; then
|
||||
HF_TOKEN_VALUE=$(cat "${HF_TOKEN_FILE}")
|
||||
fi
|
||||
|
||||
echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}"
|
||||
docker run -dt \
|
||||
--group-add 992 \
|
||||
${VIDEO_GID:+--group-add "${VIDEO_GID}"} \
|
||||
${RENDER_GID:+--group-add "${RENDER_GID}"} \
|
||||
--device /dev/dri \
|
||||
-v /dev/dri/by-path:/dev/dri/by-path \
|
||||
-v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
|
||||
-v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \
|
||||
-e HF_TOKEN="${HF_TOKEN_VALUE}" \
|
||||
--name "${CONTAINER_NAME}" \
|
||||
"${IMAGE}"
|
||||
|
||||
# Mark the workspace mount as a safe directory so git operations as root
|
||||
# inside the container don't trip the cross-user repo guard.
|
||||
docker exec "${CONTAINER_NAME}" git config --global --add safe.directory /sglang-checkout || true
|
||||
Reference in New Issue
Block a user