Files
sglang/.github/workflows/nightly-test-intel.yml

452 lines
20 KiB
YAML

name: Nightly Test (Intel)
on:
schedule:
# 22:15 IST (UTC+5:30) → 16:45 UTC daily.
- cron: '45 16 * * *'
push:
branches:
- main
paths:
- "python/sglang/version.py"
workflow_dispatch:
inputs:
continue_on_error:
description: 'Continue on error (do not fail the workflow on test failures)'
required: false
type: boolean
default: true
workflow_call:
inputs:
ref:
description: "Branch, tag or SHA to checkout"
required: false
type: string
default: ""
continue_on_error:
description: 'Continue on error (do not fail the workflow on test failures)'
required: false
type: boolean
default: true
concurrency:
group: nightly-test-intel-${{ github.event_name == 'workflow_dispatch' && format('manual-{0}', github.run_id) || inputs.ref && format('caller-{0}', github.run_id) || github.ref }}
cancel-in-progress: ${{ !inputs.ref && github.event_name != 'workflow_call' && github.event_name != 'workflow_dispatch' }}
jobs:
nightly-xpu-1-gpu:
if: github.repository == 'sgl-project/sglang'
runs-on: intel-bmg-nightly
env:
DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
steps:
- name: Reset workspace ownership
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.ref || github.sha }}
- name: Start CI container (pull intel/sglang-dev:latest)
run: |
export HF_TOKEN="$(cat ~/huggingface_token.txt)"
bash scripts/ci/xpu/xpu_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
# HF /whoami-v2 is aggressively rate-limited and the three matrix
# jobs (1/2/4-GPU) all hit it at the same second under a scheduled
# run. Retry with linear backoff + jitter to desynchronize them.
docker exec ci_sglang_xpu /bin/bash -c '
for i in 1 2 3 4 5; do
/opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0
sleep_secs=$((i * 15 + RANDOM % 15))
echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..."
sleep "${sleep_secs}"
done
exit 1
'
- name: Download olmOCR-bench dataset (for test_deepseek_ocr_2_olmbench)
timeout-minutes: 30
run: |
# PyMuPDF renders the bench PDFs to images; bench_sglang.py errors
# every sample without it.
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pymupdf
docker exec ci_sglang_xpu /bin/bash -c '
/opt/venv/bin/hf download --repo-type dataset allenai/olmOCR-bench \
--local-dir /sglang-checkout/olmOCR-bench'
- name: Nightly Test (1-GPU XPU)
timeout-minutes: 240
run: |
touch github_summary.md nightly-xpu-1-gpu-metrics.jsonl
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
OLMOCR_BENCH_DIR=/sglang-checkout/olmOCR-bench/bench_data \
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-1-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-1-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Upload per-model metrics jsonl
if: always()
uses: actions/upload-artifact@v4
with:
name: nightly-xpu-1-gpu-metrics
path: nightly-xpu-1-gpu-metrics.jsonl
if-no-files-found: warn
retention-days: 30
- name: Cleanup container
if: always()
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
find . -type f -name "*.pyc" -delete || true
docker rm -f ci_sglang_xpu || true
if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
fi
nightly-xpu-2-gpu:
if: github.repository == 'sgl-project/sglang'
runs-on: intel-bmg-nightly
env:
DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
steps:
- name: Reset workspace ownership
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.ref || github.sha }}
- name: Start CI container (pull intel/sglang-dev:latest)
run: |
export HF_TOKEN="$(cat ~/huggingface_token.txt)"
bash scripts/ci/xpu/xpu_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
# HF /whoami-v2 is aggressively rate-limited and the three matrix
# jobs (1/2/4-GPU) all hit it at the same second under a scheduled
# run. Retry with linear backoff + jitter to desynchronize them.
docker exec ci_sglang_xpu /bin/bash -c '
for i in 1 2 3 4 5; do
/opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0
sleep_secs=$((i * 15 + RANDOM % 15))
echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..."
sleep "${sleep_secs}"
done
exit 1
'
- name: Nightly Test (2-GPU XPU)
timeout-minutes: 240
run: |
touch github_summary.md nightly-xpu-2-gpu-metrics.jsonl
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-2-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Upload per-model metrics jsonl
if: always()
uses: actions/upload-artifact@v4
with:
name: nightly-xpu-2-gpu-metrics
path: nightly-xpu-2-gpu-metrics.jsonl
if-no-files-found: warn
retention-days: 30
- name: Cleanup container
if: always()
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
find . -type f -name "*.pyc" -delete || true
docker rm -f ci_sglang_xpu || true
if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
fi
nightly-xpu-4-gpu:
if: github.repository == 'sgl-project/sglang'
runs-on: intel-bmg-nightly
env:
DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
steps:
- name: Reset workspace ownership
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.ref || github.sha }}
- name: Start CI container (pull intel/sglang-dev:latest)
run: |
export HF_TOKEN="$(cat ~/huggingface_token.txt)"
bash scripts/ci/xpu/xpu_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
# HF /whoami-v2 is aggressively rate-limited and the three matrix
# jobs (1/2/4-GPU) all hit it at the same second under a scheduled
# run. Retry with linear backoff + jitter to desynchronize them.
docker exec ci_sglang_xpu /bin/bash -c '
for i in 1 2 3 4 5; do
/opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0
sleep_secs=$((i * 15 + RANDOM % 15))
echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..."
sleep "${sleep_secs}"
done
exit 1
'
- name: Nightly Test (4-GPU XPU)
timeout-minutes: 480
run: |
touch github_summary.md nightly-xpu-4-gpu-metrics.jsonl
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-4-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Upload per-model metrics jsonl
if: always()
uses: actions/upload-artifact@v4
with:
name: nightly-xpu-4-gpu-metrics
path: nightly-xpu-4-gpu-metrics.jsonl
if-no-files-found: warn
retention-days: 30
- name: Cleanup container
if: always()
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
find . -type f -name "*.pyc" -delete || true
docker rm -f ci_sglang_xpu || true
if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
fi
nightly-xpu-8-gpu:
if: github.repository == 'sgl-project/sglang'
runs-on: intel-bmg-nightly
env:
DOCKERHUB_INTEL_USERNAME: ${{ secrets.DOCKERHUB_INTEL_USERNAME }}
DOCKERHUB_INTEL_TOKEN: ${{ secrets.DOCKERHUB_INTEL_TOKEN }}
steps:
- name: Reset workspace ownership
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ inputs.ref || github.sha }}
- name: Start CI container (pull intel/sglang-dev:latest)
run: |
export HF_TOKEN="$(cat ~/huggingface_token.txt)"
bash scripts/ci/xpu/xpu_ci_start_container.sh
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Overlay in-tree sglang test helpers onto image site-packages
run: |
# Bridges the up-to-24h window between merge and nightly image rebuild.
dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang, os; print(os.path.dirname(sglang.__file__))")
docker exec ci_sglang_xpu cp \
/sglang-checkout/python/sglang/test/ci/ci_utils.py \
"$dest/test/ci/ci_utils.py"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
"import sglang.test.ci.ci_utils; print('overlay ok')"
- name: HF login + install run_suite extras
timeout-minutes: 10
run: |
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
# Diffusion extras
# Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sglang[diffusion]"
# HF /whoami-v2 is aggressively rate-limited and the three matrix
# jobs (1/2/4/8-GPU) all hit it at the same second under a scheduled
# run. Retry with linear backoff + jitter to desynchronize them.
docker exec ci_sglang_xpu /bin/bash -c '
for i in 1 2 3 4 5; do
/opt/venv/bin/hf auth login --token "${HF_TOKEN}" && exit 0
sleep_secs=$((i * 15 + RANDOM % 15))
echo "hf auth login failed (attempt $i), retrying in ${sleep_secs}s..."
sleep "${sleep_secs}"
done
exit 1
'
- name: Nightly Test (8-GPU XPU)
timeout-minutes: 480
run: |
touch github_summary.md nightly-xpu-8-gpu-metrics.jsonl
docker exec ci_sglang_xpu bash -c "
source /opt/venv/bin/activate &&
cd /sglang-checkout/test &&
GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \
SGLANG_IS_IN_CI=true \
SGLANG_TEST_METRICS_FILE=/sglang-checkout/nightly-xpu-8-gpu-metrics.jsonl \
python3 run_suite.py --hw xpu --suite nightly-xpu-8-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }}
" || TEST_EXIT_CODE=$?
echo "$(<github_summary.md)" >> $GITHUB_STEP_SUMMARY || true
exit ${TEST_EXIT_CODE:-0}
- name: Upload per-model metrics jsonl
if: always()
uses: actions/upload-artifact@v4
with:
name: nightly-xpu-8-gpu-metrics
path: nightly-xpu-8-gpu-metrics.jsonl
if-no-files-found: warn
retention-days: 30
- name: Cleanup container
if: always()
run: |
docker run --rm -v "${{ github.workspace }}:/w" busybox:latest \
chown -R "$(id -u):$(id -g)" /w || true
rm -rf test/result.jsonl test/results test/.pytest_cache .pytest_cache || true
find . -type d -name "__pycache__" -prune -exec rm -rf {} + || true
find . -type f -name "*.pyc" -delete || true
docker rm -f ci_sglang_xpu || true
if [[ -n "${CI_SGLANG_XPU_IMAGE:-}" ]]; then
docker rmi -f "${CI_SGLANG_XPU_IMAGE}" || true
fi
check-all-jobs:
if: always() && (github.repository == 'sgl-project/sglang' || github.event_name == 'workflow_dispatch')
needs:
- nightly-xpu-1-gpu
- nightly-xpu-2-gpu
- nightly-xpu-4-gpu
- nightly-xpu-8-gpu
runs-on: ubuntu-latest
steps:
- name: Check if any job failed
run: |
if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then
echo "One or more nightly test jobs failed"
exit 1
fi
if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then
echo "One or more nightly test jobs were cancelled"
exit 1
fi
echo "All nightly test jobs passed"