name: AMD CI Job Monitor

on:
  schedule:
    - cron: '0 0 * * *'  # Daily at midnight UTC
  pull_request:
    paths:
      - '.github/workflows/amd-ci-job-monitor.yml'
      - 'scripts/ci/utils/query_job_status.py'
  workflow_dispatch:
    inputs:
      hours:
        description: 'Time window in hours'
        required: false
        default: '24'
        type: string
      job_filter:
        description: 'Job name filter (leave empty for all AMD jobs)'
        required: false
        type: string

# Bound the API cost when the same ref pushes repeatedly. See note in
# runner-utilization.yml for the original incident this guards against.
concurrency:
  group: amd-ci-job-monitor-${{ github.ref }}
  cancel-in-progress: true

jobs:
  fetch-actions-data:
    name: Fetch Actions Snapshot
    # Skip fork PRs entirely, and require the `run-ci` label on same-repo PRs.
    # schedule and workflow_dispatch always run.
    if: >-
      github.event_name != 'pull_request' ||
      (github.event.pull_request.head.repo.full_name == github.repository &&
       contains(github.event.pull_request.labels.*.name, 'run-ci'))
    runs-on: ubuntu-latest
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: pip install tabulate

      - name: Select workflows for snapshot
        id: select-workflows
        run: |
          if [[ -n "${{ inputs.job_filter }}" ]]; then
            echo "workflows=pr-test-amd.yml" >> "$GITHUB_OUTPUT"
          else
            echo "workflows=pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" >> "$GITHUB_OUTPUT"
          fi

      - name: Fetch Actions data snapshot
        timeout-minutes: 30
        run: |
          # PR-trigger is just a script smoke check; scan a 20-minute window
          # (0.34h) to bound API cost. schedule / dispatch run the full report.
          python scripts/ci/utils/query_job_status.py \
            --repo ${{ github.repository }} \
            --workflow "${{ steps.select-workflows.outputs.workflows }}" \
            --hours ${{ (github.event_name == 'pull_request' && '0.34') || inputs.hours || '24' }} \
            --dump-data-file actions-job-snapshot.json

      - name: Upload Actions data snapshot
        uses: actions/upload-artifact@v4
        with:
          name: actions-job-snapshot
          path: actions-job-snapshot.json
          if-no-files-found: error

  # Single job filter mode
  custom-report:
    name: Custom Job Report
    if: ${{ inputs.job_filter }}
    needs: fetch-actions-data
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: pip install tabulate

      - name: Download Actions data snapshot
        uses: actions/download-artifact@v4
        with:
          name: actions-job-snapshot
          path: ci-data

      - name: Generate Custom Job Report
        timeout-minutes: 30
        run: |
          python scripts/ci/utils/query_job_status.py \
            --repo ${{ github.repository }} \
            --job "${{ inputs.job_filter }}" \
            --workflow "pr-test-amd.yml" \
            --hours ${{ inputs.hours || '24' }} \
            --input-data-file ci-data/actions-job-snapshot.json \
            --summary

  # Parse workflow files to get job names dynamically
  parse-workflows:
    name: Parse Workflow Jobs
    if: ${{ !inputs.job_filter }}
    runs-on: ubuntu-latest
    outputs:
      pr_jobs: ${{ steps.parse.outputs.pr_jobs }}
      nightly_jobs: ${{ steps.parse.outputs.nightly_jobs }}
      disagg_jobs: ${{ steps.parse.outputs.disagg_jobs }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Parse workflow files
        id: parse
        run: |
          set -euo pipefail

          # Parse the canonical multi-version AMD PR gate (exclude utility jobs)
          # Excluded: call-gate, check-changes, finish, cancel, check-all-jobs,
          # plus the stage waits -- they run on ubuntu-latest, so the
          # self-hosted-only snapshot has nothing for them. Keep the extra call:
          # its delegated GPU jobs use the explicit caller name as their prefix.
          pr_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/pr-test-amd.yml | \
            grep -v -E '^(call-gate|check-changes|pr-test-amd-finish|cancel|check-all-jobs|wait-for-stage-[ab]-amd)$' | \
            jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "pr_jobs=$pr_jobs" >> $GITHUB_OUTPUT
          echo "PR jobs: $pr_jobs"

          # Parse the canonical multi-version AMD nightly (exclude utility jobs)
          # Excluded: check-all-jobs
          # One nightly run covers every flavor, so ask for each by its full name:
          # a stem-only prefix matches every flavor and merges them into a single
          # row, hiding a regression that only reproduces on one image. Nightly
          # names carry the runner too, so pair each job id with its runs-on.
          # Keep in sync with the rocm_version matrix in the nightly workflow.
          rocm_flavors="rocm10 rocm724 rocm720"
          nightly_jobs=$(yq -r '.jobs | to_entries[] | .key + " " + .value."runs-on"' .github/workflows/nightly-test-amd.yml | \
            grep -v -E '^check-all-jobs ' | \
            while read -r id runner; do
              for f in $rocm_flavors; do echo "$id ($f, $runner)"; done
            done | \
            jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "nightly_jobs=$nightly_jobs" >> $GITHUB_OUTPUT
          echo "Nightly jobs: $nightly_jobs"

          # Parse nightly-amd-mi355x-disagg.yml (exclude utility jobs)
          # Excluded: setup, collect-results -- both run on ubuntu-latest and the
          # snapshot only keeps self-hosted jobs, so they would report as empty.
          disagg_jobs=$(yq -r '.jobs | keys | .[]' .github/workflows/nightly-amd-mi355x-disagg.yml | \
            grep -v -E '^(setup|collect-results)$' | \
            jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "disagg_jobs=$disagg_jobs" >> $GITHUB_OUTPUT
          echo "MI355X disagg jobs: $disagg_jobs"

  # PR CI reports using dynamic matrix
  pr-ci-reports:
    name: PR - ${{ matrix.job_name }}
    needs: [parse-workflows, fetch-actions-data]
    if: ${{ !inputs.job_filter }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        job_name: ${{ fromJson(needs.parse-workflows.outputs.pr_jobs) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: pip install tabulate

      - name: Download Actions data snapshot
        uses: actions/download-artifact@v4
        with:
          name: actions-job-snapshot
          path: ci-data

      - name: Generate Report
        timeout-minutes: 15
        run: |
          python scripts/ci/utils/query_job_status.py \
            --repo ${{ github.repository }} \
            --job "${{ matrix.job_name }}" \
            --workflow "pr-test-amd.yml" \
            --hours ${{ inputs.hours || '24' }} \
            --input-data-file ci-data/actions-job-snapshot.json \
            --summary

  # Nightly AMD test reports using dynamic matrix
  nightly-reports:
    name: Nightly - ${{ matrix.job_name }}
    needs: [parse-workflows, fetch-actions-data]
    if: ${{ !inputs.job_filter }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        job_name: ${{ fromJson(needs.parse-workflows.outputs.nightly_jobs) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: pip install tabulate

      - name: Download Actions data snapshot
        uses: actions/download-artifact@v4
        with:
          name: actions-job-snapshot
          path: ci-data

      - name: Generate Nightly Report
        timeout-minutes: 15
        run: |
          python scripts/ci/utils/query_job_status.py \
            --repo ${{ github.repository }} \
            --job "${{ matrix.job_name }}" \
            --workflow "nightly-test-amd.yml" \
            --hours ${{ inputs.hours || '24' }} \
            --input-data-file ci-data/actions-job-snapshot.json \
            --summary

  # MI355X 2N 1P1D disagg nightly reports using dynamic matrix
  nightly-disagg-reports:
    name: Nightly MI355X Disagg - ${{ matrix.job_name }}
    needs: [parse-workflows, fetch-actions-data]
    if: ${{ !inputs.job_filter }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        job_name: ${{ fromJson(needs.parse-workflows.outputs.disagg_jobs) }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: pip install tabulate

      - name: Download Actions data snapshot
        uses: actions/download-artifact@v4
        with:
          name: actions-job-snapshot
          path: ci-data

      - name: Generate MI355X Disagg Report
        timeout-minutes: 15
        run: |
          python scripts/ci/utils/query_job_status.py \
            --repo ${{ github.repository }} \
            --job "${{ matrix.job_name }}" \
            --workflow "nightly-amd-mi355x-disagg.yml" \
            --hours ${{ inputs.hours || '24' }} \
            --input-data-file ci-data/actions-job-snapshot.json \
            --summary

  # Runner fleet report - cross-workflow runner analytics in a single pass
  runner-fleet-report:
    name: Runner Fleet Report
    if: ${{ !inputs.job_filter }}
    needs: fetch-actions-data
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: pip install tabulate

      - name: Download Actions data snapshot
        uses: actions/download-artifact@v4
        with:
          name: actions-job-snapshot
          path: ci-data

      - name: Generate Runner Fleet Report
        timeout-minutes: 30
        run: |
          python scripts/ci/utils/query_job_status.py \
            --repo ${{ github.repository }} \
            --runner-report \
            --workflow "pr-test-amd.yml,nightly-test-amd.yml,nightly-amd-mi355x-disagg.yml" \
            --hours ${{ inputs.hours || '24' }} \
            --input-data-file ci-data/actions-job-snapshot.json \
            --summary