diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index 2cbcc8b6f..867d03f3a 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -73,57 +73,52 @@ jobs: needs: [check-changes, pr-gate] if: needs.check-changes.outputs.main_package == 'true' runs-on: intel-bmg + env: + DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }} + DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }} steps: + - name: Reset workspace ownership + run: | + # Prior runs install sglang as root inside the CI container against + # the bind-mounted workspace, leaving root-owned egg-info on the host. + # actions/checkout's pre-clean step can't unlink those as the runner + # user, so reset ownership before checkout runs. + docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \ + chown -R "$(id -u):$(id -g)" /w || true + - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ inputs.ref || github.ref }} - - name: Build Docker image + - name: Start CI container run: | - PR_REPO=${{ github.event.pull_request.head.repo.clone_url }} - PR_HEAD_REF=${{ github.head_ref }} - docker build \ - ${PR_REPO:+--build-arg SG_LANG_REPO=$PR_REPO} \ - ${PR_HEAD_REF:+--build-arg SG_LANG_BRANCH=$PR_HEAD_REF} \ - --no-cache --progress=plain -f docker/xpu.Dockerfile -t xpu_sglang_main:bmg . - - - name: Run container - id: start_container - run: | - container_id=$(docker run -dt \ - --group-add 992 \ - --group-add $(getent group video | cut -d: -f3) \ - --group-add $(getent group render | cut -d: -f3) \ - -v $HOME/.cache/huggingface:/root/.cache/huggingface \ - --device /dev/dri \ - -v /dev/dri/by-path:/dev/dri/by-path \ - -e HF_TOKEN="$(cat ~/huggingface_token.txt)" \ - xpu_sglang_main:bmg) - echo "Started container: $container_id" - echo "container_id=$container_id" >> "$GITHUB_OUTPUT" + export HF_TOKEN="$(cat ~/huggingface_token.txt)" + bash scripts/ci/xpu/xpu_ci_start_container.sh + env: + GITHUB_WORKSPACE: ${{ github.workspace }} - name: Install Dependency - timeout-minutes: 20 + timeout-minutes: 60 run: | - cid="${{ steps.start_container.outputs.container_id }}" - docker exec "$cid" /opt/venv/bin/python3 -m pip install --upgrade pip - docker exec "$cid" /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate - docker exec "$cid" /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python - docker exec "$cid" /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN} ' + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --upgrade pip + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang + docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml + docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 + docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install triton-xpu==3.7.1 --index-url https://download.pytorch.org/whl/test/xpu --force-reinstall + docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' - name: Run stage-a tests timeout-minutes: 30 run: | - cid="${{ steps.start_container.outputs.container_id }}" - docker exec "$cid" bash -c "source /opt/venv/bin/activate && cd /sgl-workspace/sglang/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu" + docker exec ci_sglang_xpu bash -c "source /opt/venv/bin/activate && cd /sglang-checkout/test && python3 run_suite.py --hw xpu --suite stage-a-test-1-gpu-xpu" - name: Cleanup container if: always() - run: | - cid="${{ steps.start_container.outputs.container_id }}" - docker rm -f "$cid" || true + run: docker rm -f ci_sglang_xpu || true # ==================== Wait for Stage A ==================== # wait-for-stage-a: diff --git a/scripts/ci/xpu/xpu_ci_start_container.sh b/scripts/ci/xpu/xpu_ci_start_container.sh new file mode 100755 index 000000000..0847168b0 --- /dev/null +++ b/scripts/ci/xpu/xpu_ci_start_container.sh @@ -0,0 +1,159 @@ +#!/bin/bash +set -euo pipefail + +# Start the Intel XPU CI container (ci_sglang_xpu) using the latest nightly +# intel/sglang-dev image published by .github/workflows/release-docker-intel-xpu-nightly.yml. +# +# Walks back N days through nightly date-stamped tags, pulls the first match, +# then starts a long-running container that subsequent steps `docker exec` into. + +CONTAINER_NAME="ci_sglang_xpu" +IMAGE_REPO="intel/sglang-dev" +TAG_PREFIX="nightly-dev-xpu-bmg" +LOOKBACK_DAYS=7 +CUSTOM_IMAGE="" + +while [[ $# -gt 0 ]]; do + case $1 in + --custom-image) CUSTOM_IMAGE="$2"; shift 2;; + --container-name) CONTAINER_NAME="$2"; shift 2;; + --lookback-days) LOOKBACK_DAYS="$2"; shift 2;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --custom-image IMAGE Use a specific Docker image directly" + echo " --container-name NAME Override container name (default: ${CONTAINER_NAME})" + echo " --lookback-days N Days of nightly tags to scan (default: ${LOOKBACK_DAYS})" + exit 0 + ;; + *) echo "Unknown option $1"; exit 1;; + esac +done + +# Retry a command with exponential backoff. Usage: retry_with_backoff +retry_with_backoff() { + local max_attempts=$1; shift + local attempt=1 + local wait_secs=30 + local jitter=$(( RANDOM % 30 )) + while true; do + if "$@"; then + return 0 + fi + if (( attempt >= max_attempts )); then + echo "Error: '$*' failed after ${max_attempts} attempts" >&2 + return 1 + fi + local sleep_time=$(( wait_secs + jitter )) + echo "Attempt ${attempt}/${max_attempts} failed. Retrying in ${sleep_time}s..." >&2 + sleep "${sleep_time}" + (( attempt++ )) + (( wait_secs = wait_secs * 2 > 300 ? 300 : wait_secs * 2 )) + jitter=$(( RANDOM % 30 )) + done +} + +# Authenticate to Docker Hub when credentials are present (avoids anonymous pull +# rate limits). Both vars are optional; falls back to unauthenticated pulls. +if [[ -n "${DOCKERHUB_INTEL_USERNAME:-}" && -n "${DOCKERHUB_INTEL_TOKEN:-}" ]]; then + echo "Logging in to Docker Hub..." + if retry_with_backoff 6 sh -c 'echo "${DOCKERHUB_INTEL_TOKEN}" | docker login -u "${DOCKERHUB_INTEL_USERNAME}" --password-stdin >/dev/null 2>&1'; then + echo "Docker Hub login successful" + else + echo "Warning: Docker Hub login failed after retries; continuing with unauthenticated pulls" >&2 + fi +fi + +# Locate the latest published nightly image. Walks back LOOKBACK_DAYS days of +# date-stamped tags (commit-hash suffix is unknown, so we query Docker Hub). +find_latest_image() { + local days_back target_date matched_tag + + # Local cache first. + for (( days_back=0; days_back&2 + echo "${matched_tag}" + return 0 + fi + done + + # Then query Docker Hub for any tag matching the date prefix. + for (( days_back=0; days_back&2 + matched_tag=$(curl -fsSL \ + "https://registry.hub.docker.com/v2/repositories/${IMAGE_REPO}/tags?page_size=100&name=${TAG_PREFIX}-${target_date}" \ + 2>/dev/null \ + | grep -o '"name":"[^"]*"' | cut -d'"' -f4 | head -n 1 || true) + if [[ -n "${matched_tag}" ]]; then + echo "Found published image: ${IMAGE_REPO}:${matched_tag}" >&2 + echo "${IMAGE_REPO}:${matched_tag}" + return 0 + fi + done + + # Final fallback: any locally cached image matching the prefix. + matched_tag=$(docker images --format '{{.Repository}}:{{.Tag}}' \ + --filter "reference=${IMAGE_REPO}:${TAG_PREFIX}-*" \ + | sort -r | head -n 1) + if [[ -n "${matched_tag}" ]]; then + echo "Using cached fallback image: ${matched_tag}" >&2 + echo "${matched_tag}" + return 0 + fi + + echo "Error: no ${IMAGE_REPO}:${TAG_PREFIX}-* image found in the last ${LOOKBACK_DAYS} days" >&2 + return 1 +} + +if [[ -n "${CUSTOM_IMAGE}" ]]; then + IMAGE="${CUSTOM_IMAGE}" + echo "Using custom image: ${IMAGE}" + retry_with_backoff 6 docker pull "${IMAGE}" +else + IMAGE=$(find_latest_image) + if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then + retry_with_backoff 6 docker pull "${IMAGE}" + else + echo "Image ${IMAGE} already present locally; skipping pull" + fi +fi + +# Remove any stale container of the same name so re-runs are idempotent. +if docker ps -a --format '{{.Names}}' | grep -qx "${CONTAINER_NAME}"; then + echo "Removing existing container: ${CONTAINER_NAME}" + docker rm -f "${CONTAINER_NAME}" >/dev/null +fi + +VIDEO_GID=$(getent group video | cut -d: -f3) +RENDER_GID=$(getent group render | cut -d: -f3) + +HF_TOKEN_FILE="${HOME}/huggingface_token.txt" +HF_TOKEN_VALUE="" +if [[ -n "${HF_TOKEN:-}" ]]; then + HF_TOKEN_VALUE="${HF_TOKEN}" +elif [[ -r "${HF_TOKEN_FILE}" ]]; then + HF_TOKEN_VALUE=$(cat "${HF_TOKEN_FILE}") +fi + +echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}" +docker run -dt \ + --group-add 992 \ + ${VIDEO_GID:+--group-add "${VIDEO_GID}"} \ + ${RENDER_GID:+--group-add "${RENDER_GID}"} \ + --device /dev/dri \ + -v /dev/dri/by-path:/dev/dri/by-path \ + -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \ + -v "${GITHUB_WORKSPACE:-$PWD}:/sglang-checkout" \ + -e HF_TOKEN="${HF_TOKEN_VALUE}" \ + --name "${CONTAINER_NAME}" \ + "${IMAGE}" + +# Mark the workspace mount as a safe directory so git operations as root +# inside the container don't trip the cross-user repo guard. +docker exec "${CONTAINER_NAME}" git config --global --add safe.directory /sglang-checkout || true