Add XPU CI job monitor workflow (#29807)
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
ashwini rathi
co-authored by
Claude Opus 4.7
Claude Opus 4.7
parent
2b3223a953
commit
75cdaf432a
@@ -0,0 +1,199 @@
|
|||||||
|
name: XPU CI Job Monitor
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: '0 0 * * *' # Daily at midnight UTC
|
||||||
|
pull_request:
|
||||||
|
paths:
|
||||||
|
- '.github/workflows/xpu-ci-job-monitor.yml'
|
||||||
|
- 'scripts/ci/utils/xpu_job_monitor.py'
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
hours:
|
||||||
|
description: 'Time window in hours'
|
||||||
|
required: false
|
||||||
|
default: '24'
|
||||||
|
type: string
|
||||||
|
job_filter:
|
||||||
|
description: 'Job name filter (leave empty for all XPU jobs)'
|
||||||
|
required: false
|
||||||
|
type: string
|
||||||
|
|
||||||
|
# Bound the API cost when the same ref pushes repeatedly. See note in
|
||||||
|
# runner-utilization.yml for the original incident this guards against.
|
||||||
|
concurrency:
|
||||||
|
group: xpu-ci-job-monitor-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
fetch-actions-data:
|
||||||
|
name: Fetch Actions Snapshot
|
||||||
|
# Skip fork PRs entirely, and require the `run-ci` label on same-repo PRs.
|
||||||
|
# schedule and workflow_dispatch always run.
|
||||||
|
if: >-
|
||||||
|
github.event_name != 'pull_request' ||
|
||||||
|
(github.event.pull_request.head.repo.full_name == github.repository &&
|
||||||
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
env:
|
||||||
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pip install tabulate
|
||||||
|
|
||||||
|
- name: Fetch Actions data snapshot
|
||||||
|
timeout-minutes: 30
|
||||||
|
run: |
|
||||||
|
# PR-trigger is just a script smoke check; scan a 20-minute window
|
||||||
|
# (0.34h) to bound API cost. schedule / dispatch run the full report.
|
||||||
|
python scripts/ci/utils/xpu_job_monitor.py \
|
||||||
|
--repo ${{ github.repository }} \
|
||||||
|
--workflow "pr-test-xpu.yml" \
|
||||||
|
--hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
|
||||||
|
--dump-data-file actions-job-snapshot.json
|
||||||
|
|
||||||
|
- name: Upload Actions data snapshot
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: actions-job-snapshot
|
||||||
|
path: actions-job-snapshot.json
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
# Single job filter mode
|
||||||
|
custom-report:
|
||||||
|
name: Custom Job Report
|
||||||
|
if: ${{ inputs.job_filter }}
|
||||||
|
needs: fetch-actions-data
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pip install tabulate
|
||||||
|
|
||||||
|
- name: Download Actions data snapshot
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: actions-job-snapshot
|
||||||
|
path: ci-data
|
||||||
|
|
||||||
|
- name: Generate Custom Job Report
|
||||||
|
timeout-minutes: 30
|
||||||
|
run: |
|
||||||
|
python scripts/ci/utils/xpu_job_monitor.py \
|
||||||
|
--repo ${{ github.repository }} \
|
||||||
|
--job "${{ inputs.job_filter }}" \
|
||||||
|
--workflow "pr-test-xpu.yml" \
|
||||||
|
--hours ${{ inputs.hours || '24' }} \
|
||||||
|
--input-data-file ci-data/actions-job-snapshot.json \
|
||||||
|
--summary
|
||||||
|
|
||||||
|
# Parse workflow files to get job names dynamically
|
||||||
|
parse-workflows:
|
||||||
|
name: Parse Workflow Jobs
|
||||||
|
if: ${{ !inputs.job_filter }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
outputs:
|
||||||
|
pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Parse workflow files
|
||||||
|
id: parse
|
||||||
|
run: |
|
||||||
|
# Parse pr-test-xpu.yml and extract job names (exclude utility jobs)
|
||||||
|
# Excluded: check-changes, pr-gate, wait-for-stage-a, finish
|
||||||
|
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-xpu.yml | \
|
||||||
|
grep -v -E '^(check-changes|pr-gate|wait-for-stage-a|finish)$' | \
|
||||||
|
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
||||||
|
echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
|
||||||
|
echo "PR jobs: $pr_jobs"
|
||||||
|
|
||||||
|
# PR CI reports using dynamic matrix
|
||||||
|
pr-ci-reports:
|
||||||
|
name: PR - ${{ matrix.job_name }}
|
||||||
|
needs: [parse-workflows, fetch-actions-data]
|
||||||
|
if: ${{ !inputs.job_filter }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_jobs) }}
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pip install tabulate
|
||||||
|
|
||||||
|
- name: Download Actions data snapshot
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: actions-job-snapshot
|
||||||
|
path: ci-data
|
||||||
|
|
||||||
|
- name: Generate Report
|
||||||
|
timeout-minutes: 15
|
||||||
|
run: |
|
||||||
|
python scripts/ci/utils/xpu_job_monitor.py \
|
||||||
|
--repo ${{ github.repository }} \
|
||||||
|
--job "${{ matrix.job_name }}" \
|
||||||
|
--workflow "pr-test-xpu.yml" \
|
||||||
|
--hours ${{ inputs.hours || '24' }} \
|
||||||
|
--input-data-file ci-data/actions-job-snapshot.json \
|
||||||
|
--summary
|
||||||
|
|
||||||
|
# Runner fleet report - cross-workflow runner analytics in a single pass
|
||||||
|
runner-fleet-report:
|
||||||
|
name: Runner Fleet Report
|
||||||
|
if: ${{ !inputs.job_filter }}
|
||||||
|
needs: fetch-actions-data
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Set up Python
|
||||||
|
uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.10'
|
||||||
|
|
||||||
|
- name: Install dependencies
|
||||||
|
run: pip install tabulate
|
||||||
|
|
||||||
|
- name: Download Actions data snapshot
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
name: actions-job-snapshot
|
||||||
|
path: ci-data
|
||||||
|
|
||||||
|
- name: Generate Runner Fleet Report
|
||||||
|
timeout-minutes: 30
|
||||||
|
run: |
|
||||||
|
python scripts/ci/utils/xpu_job_monitor.py \
|
||||||
|
--repo ${{ github.repository }} \
|
||||||
|
--runner-report \
|
||||||
|
--workflow "pr-test-xpu.yml" \
|
||||||
|
--hours ${{ inputs.hours || '24' }} \
|
||||||
|
--input-data-file ci-data/actions-job-snapshot.json \
|
||||||
|
--summary
|
||||||
Executable
+1947
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user