From cc7d7ba3ddd8ec59824afb9a3c7a289d300d3b46 Mon Sep 17 00:00:00 2001 From: ashwini rathi Date: Mon, 6 Jul 2026 09:56:28 +0530 Subject: [PATCH] [Intel XPU] Bump nightly per-file timeout to 120m and extend XPU CI monitor (#30100) --- .github/workflows/nightly-test-intel.yml | 4 +- .github/workflows/xpu-ci-job-monitor.yml | 61 +++++++++++++++++++++++- scripts/ci/xpu/xpu_ci_start_container.sh | 1 + 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-test-intel.yml b/.github/workflows/nightly-test-intel.yml index d68838ecf..37817a80e 100644 --- a/.github/workflows/nightly-test-intel.yml +++ b/.github/workflows/nightly-test-intel.yml @@ -84,7 +84,7 @@ jobs: source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ - python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 1800 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + python3 run_suite.py --hw xpu --suite nightly-xpu-2-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} @@ -143,7 +143,7 @@ jobs: source /opt/venv/bin/activate && cd /sglang-checkout/test && GITHUB_STEP_SUMMARY=/sglang-checkout/github_summary.md \ - python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 2400 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} + python3 run_suite.py --hw xpu --suite nightly-xpu-4-gpu --nightly --timeout-per-file 7200 ${{ (github.event_name == 'schedule' || inputs.continue_on_error) && '--continue-on-error' || '' }} " || TEST_EXIT_CODE=$? echo "$(> $GITHUB_STEP_SUMMARY || true exit ${TEST_EXIT_CODE:-0} diff --git a/.github/workflows/xpu-ci-job-monitor.yml b/.github/workflows/xpu-ci-job-monitor.yml index f1ab549d3..460a64e92 100644 --- a/.github/workflows/xpu-ci-job-monitor.yml +++ b/.github/workflows/xpu-ci-job-monitor.yml @@ -49,6 +49,15 @@ jobs: - name: Install dependencies run: pip install tabulate + - name: Select workflows for snapshot + id: select-workflows + run: | + if [[ -n "${{ inputs.job_filter }}" ]]; then + echo "workflows=pr-test-xpu.yml" >> "$GITHUB_OUTPUT" + else + echo "workflows=pr-test-xpu.yml,nightly-test-intel.yml" >> "$GITHUB_OUTPUT" + fi + - name: Fetch Actions data snapshot timeout-minutes: 30 run: | @@ -56,7 +65,7 @@ jobs: # (0.34h) to bound API cost. schedule / dispatch run the full report. python scripts/ci/utils/xpu_job_monitor.py \ --repo ${{ github.repository }} \ - --workflow "pr-test-xpu.yml" \ + --workflow "${{ steps.select-workflows.outputs.workflows }}" \ --hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \ --dump-data-file actions-job-snapshot.json @@ -109,6 +118,7 @@ jobs: runs-on: ubuntu-latest outputs: pr_jobs: ${{ steps.parse.outputs.pr_jobs }} + nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -124,6 +134,14 @@ jobs: echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT echo "PR jobs: $pr_jobs" + # Parse nightly-test-intel.yml and extract job names (exclude utility jobs) + # Excluded: check-all-jobs + nightly_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-test-intel.yml | \ + grep -v -E '^(check-all-jobs)$' | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT + echo "Nightly jobs: $nightly_jobs" + # PR CI reports using dynamic matrix pr-ci-reports: name: PR - ${{ matrix.job_name }} @@ -163,6 +181,45 @@ jobs: --input-data-file ci-data/actions-job-snapshot.json \ --summary + # Nightly XPU test reports using dynamic matrix + nightly-reports: + name: Nightly - ${{ matrix.job_name }} + needs: [parse-workflows, fetch-actions-data] + if: ${{ !inputs.job_filter }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_jobs) }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install tabulate + + - name: Download Actions data snapshot + uses: actions/download-artifact@v4 + with: + name: actions-job-snapshot + path: ci-data + + - name: Generate Nightly Report + timeout-minutes: 15 + run: | + python scripts/ci/utils/xpu_job_monitor.py \ + --repo ${{ github.repository }} \ + --job "${{ matrix.job_name }}" \ + --workflow "nightly-test-intel.yml" \ + --hours ${{ inputs.hours || '24' }} \ + --input-data-file ci-data/actions-job-snapshot.json \ + --summary + # Runner fleet report - cross-workflow runner analytics in a single pass runner-fleet-report: name: Runner Fleet Report @@ -193,7 +250,7 @@ jobs: python scripts/ci/utils/xpu_job_monitor.py \ --repo ${{ github.repository }} \ --runner-report \ - --workflow "pr-test-xpu.yml" \ + --workflow "pr-test-xpu.yml,nightly-test-intel.yml" \ --hours ${{ inputs.hours || '24' }} \ --input-data-file ci-data/actions-job-snapshot.json \ --summary diff --git a/scripts/ci/xpu/xpu_ci_start_container.sh b/scripts/ci/xpu/xpu_ci_start_container.sh index aa25e0d4b..4f75dca42 100755 --- a/scripts/ci/xpu/xpu_ci_start_container.sh +++ b/scripts/ci/xpu/xpu_ci_start_container.sh @@ -99,6 +99,7 @@ fi echo "Launching container: ${CONTAINER_NAME} from ${IMAGE}" docker run -dt \ + --shm-size 8g \ --group-add 992 \ ${VIDEO_GID:+--group-add "${VIDEO_GID}"} \ ${RENDER_GID:+--group-add "${RENDER_GID}"} \