320 lines
11 KiB
YAML
320 lines
11 KiB
YAML
name: AMD CI Job Monitor
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '0 0 * * *' # Daily at midnight UTC
|
|
pull_request:
|
|
paths:
|
|
- '.github/workflows/amd-ci-job-monitor.yml'
|
|
- 'scripts/ci/utils/query_job_status.py'
|
|
workflow_dispatch:
|
|
inputs:
|
|
hours:
|
|
description: 'Time window in hours'
|
|
required: false
|
|
default: '24'
|
|
type: string
|
|
job_filter:
|
|
description: 'Job name filter (leave empty for all AMD jobs)'
|
|
required: false
|
|
type: string
|
|
|
|
# Bound the API cost when the same ref pushes repeatedly. See note in
|
|
# runner-utilization.yml for the original incident this guards against.
|
|
concurrency:
|
|
group: amd-ci-job-monitor-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
fetch-actions-data:
|
|
name: Fetch Actions Snapshot
|
|
# Skip fork PRs entirely, and require the `run-ci` label on same-repo PRs.
|
|
# schedule and workflow_dispatch always run.
|
|
if: >-
|
|
github.event_name != 'pull_request' ||
|
|
(github.event.pull_request.head.repo.full_name == github.repository &&
|
|
contains(github.event.pull_request.labels.*.name, 'run-ci'))
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: pip install tabulate
|
|
|
|
- name: Select workflows for snapshot
|
|
id: select-workflows
|
|
run: |
|
|
if [[ -n "${{ inputs.job_filter }}" ]]; then
|
|
echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
- name: Fetch Actions data snapshot
|
|
timeout-minutes: 30
|
|
run: |
|
|
# PR-trigger is just a script smoke check; scan a 20-minute window
|
|
# (0.34h) to bound API cost. schedule / dispatch run the full report.
|
|
python scripts/ci/utils/query_job_status.py \
|
|
--repo ${{ github.repository }} \
|
|
--workflow "${{ steps.select-workflows.outputs.workflows }}" \
|
|
--hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
|
|
--dump-data-file actions-job-snapshot.json
|
|
|
|
- name: Upload Actions data snapshot
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: actions-job-snapshot
|
|
path: actions-job-snapshot.json
|
|
if-no-files-found: error
|
|
|
|
# Single job filter mode
|
|
custom-report:
|
|
name: Custom Job Report
|
|
if: ${{ inputs.job_filter }}
|
|
needs: fetch-actions-data
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: pip install tabulate
|
|
|
|
- name: Download Actions data snapshot
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: actions-job-snapshot
|
|
path: ci-data
|
|
|
|
- name: Generate Custom Job Report
|
|
timeout-minutes: 30
|
|
run: |
|
|
python scripts/ci/utils/query_job_status.py \
|
|
--repo ${{ github.repository }} \
|
|
--job "${{ inputs.job_filter }}" \
|
|
--workflow "pr-test-amd.yml" \
|
|
--hours ${{ inputs.hours || '24' }} \
|
|
--input-data-file ci-data/actions-job-snapshot.json \
|
|
--summary
|
|
|
|
# Parse workflow files to get job names dynamically
|
|
parse-workflows:
|
|
name: Parse Workflow Jobs
|
|
if: ${{ !inputs.job_filter }}
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
|
|
nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
|
|
disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Parse workflow files
|
|
id: parse
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
# Parse the canonical multi-version AMD PR gate (exclude utility jobs)
|
|
# Excluded: call-gate, check-changes, finish, cancel, check-all-jobs,
|
|
# plus the stage waits -- they run on ubuntu-latest, so the
|
|
# self-hosted-only snapshot has nothing for them. Keep the extra call:
|
|
# its delegated GPU jobs use the explicit caller name as their prefix.
|
|
pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
|
|
grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd)$' | \
|
|
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
|
echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
|
|
echo "PR jobs: $pr_jobs"
|
|
|
|
# Parse the canonical multi-version AMD nightly (exclude utility jobs)
|
|
# Excluded: check-all-jobs
|
|
# One nightly run covers every flavor, so ask for each by its full name:
|
|
# a stem-only prefix matches every flavor and merges them into a single
|
|
# row, hiding a regression that only reproduces on one image. Nightly
|
|
# names carry the runner too, so pair each job id with its runs-on.
|
|
# Keep in sync with the rocm_version matrix in the nightly workflow.
|
|
rocm_flavors="rocm10 rocm724 rocm720"
|
|
nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd.yml | \
|
|
grep -v -E '^check-all-jobs ' | \
|
|
while read -r id runner; do
|
|
for f in $rocm_flavors; do echo "$id ($f, $runner)"; done
|
|
done | \
|
|
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
|
echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
|
|
echo "Nightly jobs: $nightly_jobs"
|
|
|
|
# Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs)
|
|
# Excluded: setup, collect-results -- both run on ubuntu-latest and the
|
|
# snapshot only keeps self-hosted jobs, so they would report as empty.
|
|
disagg_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-amd-mi355x-disagg.yml | \
|
|
grep -v -E '^(setup|collect-results)$' | \
|
|
jq -R -s -c 'split("\n") | map(select(length > 0))')
|
|
echo "disagg_jobs=$disagg_jobs" >> $GITHUB_OUTPUT
|
|
echo "MI355X disagg jobs: $disagg_jobs"
|
|
|
|
# PR CI reports using dynamic matrix
|
|
pr-ci-reports:
|
|
name: PR - ${{ matrix.job_name }}
|
|
needs: [parse-workflows, fetch-actions-data]
|
|
if: ${{ !inputs.job_filter }}
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_jobs) }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: pip install tabulate
|
|
|
|
- name: Download Actions data snapshot
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: actions-job-snapshot
|
|
path: ci-data
|
|
|
|
- name: Generate Report
|
|
timeout-minutes: 15
|
|
run: |
|
|
python scripts/ci/utils/query_job_status.py \
|
|
--repo ${{ github.repository }} \
|
|
--job "${{ matrix.job_name }}" \
|
|
--workflow "pr-test-amd.yml" \
|
|
--hours ${{ inputs.hours || '24' }} \
|
|
--input-data-file ci-data/actions-job-snapshot.json \
|
|
--summary
|
|
|
|
# Nightly AMD test reports using dynamic matrix
|
|
nightly-reports:
|
|
name: Nightly - ${{ matrix.job_name }}
|
|
needs: [parse-workflows, fetch-actions-data]
|
|
if: ${{ !inputs.job_filter }}
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_jobs) }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: pip install tabulate
|
|
|
|
- name: Download Actions data snapshot
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: actions-job-snapshot
|
|
path: ci-data
|
|
|
|
- name: Generate Nightly Report
|
|
timeout-minutes: 15
|
|
run: |
|
|
python scripts/ci/utils/query_job_status.py \
|
|
--repo ${{ github.repository }} \
|
|
--job "${{ matrix.job_name }}" \
|
|
--workflow "nightly-test-amd.yml" \
|
|
--hours ${{ inputs.hours || '24' }} \
|
|
--input-data-file ci-data/actions-job-snapshot.json \
|
|
--summary
|
|
|
|
# MI355X 2N 1P1D disagg nightly reports using dynamic matrix
|
|
nightly-disagg-reports:
|
|
name: Nightly MI355X Disagg - ${{ matrix.job_name }}
|
|
needs: [parse-workflows, fetch-actions-data]
|
|
if: ${{ !inputs.job_filter }}
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
job_name: ${{ fromJson(needs.parse-workflows.outputs.disagg_jobs) }}
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: pip install tabulate
|
|
|
|
- name: Download Actions data snapshot
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: actions-job-snapshot
|
|
path: ci-data
|
|
|
|
- name: Generate MI355X Disagg Report
|
|
timeout-minutes: 15
|
|
run: |
|
|
python scripts/ci/utils/query_job_status.py \
|
|
--repo ${{ github.repository }} \
|
|
--job "${{ matrix.job_name }}" \
|
|
--workflow "nightly-amd-mi355x-disagg.yml" \
|
|
--hours ${{ inputs.hours || '24' }} \
|
|
--input-data-file ci-data/actions-job-snapshot.json \
|
|
--summary
|
|
|
|
# Runner fleet report - cross-workflow runner analytics in a single pass
|
|
runner-fleet-report:
|
|
name: Runner Fleet Report
|
|
if: ${{ !inputs.job_filter }}
|
|
needs: fetch-actions-data
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install dependencies
|
|
run: pip install tabulate
|
|
|
|
- name: Download Actions data snapshot
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: actions-job-snapshot
|
|
path: ci-data
|
|
|
|
- name: Generate Runner Fleet Report
|
|
timeout-minutes: 30
|
|
run: |
|
|
python scripts/ci/utils/query_job_status.py \
|
|
--repo ${{ github.repository }} \
|
|
--runner-report \
|
|
--workflow "pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" \
|
|
--hours ${{ inputs.hours || '24' }} \
|
|
--input-data-file ci-data/actions-job-snapshot.json \
|
|
--summary
|