From ae5f8c94b741407707ca42e5504a12dcd0664494 Mon Sep 17 00:00:00 2001 From: ashwini rathi Date: Thu, 6 Aug 2026 05:55:53 +0530 Subject: [PATCH] ci(xpu): harden nightly + PR XPU CI (HF login / tag fetch / docker push retries) (#32390) --- .github/workflows/nightly-test-intel.yml | 39 +++++++++++++++++-- .github/workflows/pr-test-xpu.yml | 6 +++ .../release-docker-intel-xpu-nightly.yml | 12 +++++- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly-test-intel.yml b/.github/workflows/nightly-test-intel.yml index 2bb9e3a9f..2d16a5df9 100644 --- a/.github/workflows/nightly-test-intel.yml +++ b/.github/workflows/nightly-test-intel.yml @@ -69,7 +69,18 @@ jobs: # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo). # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion]. docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]" - docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' + # HF /whoami-v2 is aggressively rate-limited and the three matrix + # jobs (1/2/4-GPU) all hit it at the same second under a scheduled + # run. Retry with linear backoff + jitter to desynchronize them. + docker exec ci_sglang_xpu /bin/bash -c ' + for i in 1 2 3 4 5; do + /opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0 + sleep_secs=$((i * 15 + RANDOM % 15)) + echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..." + sleep "${sleep_secs}" + done + exit 1 + ' - name: Download olmOCR-bench dataset (for test_deepseek_ocr_2_olmbench) timeout-minutes: 30 @@ -143,7 +154,18 @@ jobs: # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo). # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion]. docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]" - docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' + # HF /whoami-v2 is aggressively rate-limited and the three matrix + # jobs (1/2/4-GPU) all hit it at the same second under a scheduled + # run. Retry with linear backoff + jitter to desynchronize them. + docker exec ci_sglang_xpu /bin/bash -c ' + for i in 1 2 3 4 5; do + /opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0 + sleep_secs=$((i * 15 + RANDOM % 15)) + echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..." + sleep "${sleep_secs}" + done + exit 1 + ' - name: Nightly Test (2-GPU XPU) timeout-minutes: 240 @@ -206,7 +228,18 @@ jobs: # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo). # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion]. docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]" - docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' + # HF /whoami-v2 is aggressively rate-limited and the three matrix + # jobs (1/2/4-GPU) all hit it at the same second under a scheduled + # run. Retry with linear backoff + jitter to desynchronize them. + docker exec ci_sglang_xpu /bin/bash -c ' + for i in 1 2 3 4 5; do + /opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0 + sleep_secs=$((i * 15 + RANDOM % 15)) + echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..." + sleep "${sleep_secs}" + done + exit 1 + ' - name: Nightly Test (4-GPU XPU) timeout-minutes: 480 diff --git a/.github/workflows/pr-test-xpu.yml b/.github/workflows/pr-test-xpu.yml index 5c4adecca..48ac12f22 100644 --- a/.github/workflows/pr-test-xpu.yml +++ b/.github/workflows/pr-test-xpu.yml @@ -106,6 +106,9 @@ jobs: docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate "lmcache>=0.3.9" docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml + # Fetch tags so setuptools_scm resolves a real version instead of + # falling back to 0.0.0 on a shallow/tag-less checkout. + docker exec -w /sglang-checkout ci_sglang_xpu git fetch origin '+refs/tags/*:refs/tags/*' --force docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' @@ -195,6 +198,9 @@ jobs: docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install pytest expecttest ray huggingface_hub tabulate "lmcache>=0.3.9" docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip uninstall -y flashinfer-python sgl-kernel sglang docker exec ci_sglang_xpu cp /sglang-checkout/python/pyproject_xpu.toml /sglang-checkout/python/pyproject.toml + # Fetch tags so setuptools_scm resolves a real version instead of + # falling back to 0.0.0 on a shallow/tag-less checkout. + docker exec -w /sglang-checkout ci_sglang_xpu git fetch origin '+refs/tags/*:refs/tags/*' --force docker exec -w /sglang-checkout/python ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir . --extra-index-url https://download.pytorch.org/whl/xpu docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33 docker exec ci_sglang_xpu /bin/bash -c '/opt/venv/bin/hf auth login --token ${HF_TOKEN}' diff --git a/.github/workflows/release-docker-intel-xpu-nightly.yml b/.github/workflows/release-docker-intel-xpu-nightly.yml index 1daef946c..73cb4baac 100644 --- a/.github/workflows/release-docker-intel-xpu-nightly.yml +++ b/.github/workflows/release-docker-intel-xpu-nightly.yml @@ -55,5 +55,13 @@ jobs: - name: Push intel/sglang-dev run: | - docker push "intel/sglang-dev:${{ env.IMAGE_TAG }}" - docker push "intel/sglang-dev:latest" + push_with_retry() { + for i in 1 2 3 4 5; do + docker push "$1" && return 0 + echo "push failed (attempt $i), retrying in $((i*15))s..." + sleep $((i*15)) + done + return 1 + } + push_with_retry "intel/sglang-dev:${{ env.IMAGE_TAG }}" + push_with_retry "intel/sglang-dev:latest"