109 lines
5.4 KiB
YAML
109 lines
5.4 KiB
YAML
name: Check PR Test Health
|
|
description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fail-fast` label (or `highest-priority`, which implies it).
|
|
|
|
inputs:
|
|
github-token:
|
|
description: 'GitHub token for API calls'
|
|
required: false
|
|
default: ${{ github.token }}
|
|
|
|
runs:
|
|
using: composite
|
|
steps:
|
|
- name: Check PR test health
|
|
uses: actions/github-script@v8
|
|
env:
|
|
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
|
|
with:
|
|
github-token: ${{ inputs.github-token }}
|
|
script: |
|
|
core.info(`[health-check] START -- event=${context.eventName}, runId=${context.runId}`);
|
|
|
|
// Skip when explicitly requested via env var (e.g. release branch cut)
|
|
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
|
|
core.info('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
|
|
return;
|
|
}
|
|
|
|
// Skip for scheduled runs -- they should collect all failures, not fail-fast
|
|
if (context.eventName === 'schedule') {
|
|
core.info('[health-check] SKIP: scheduled run');
|
|
return;
|
|
}
|
|
|
|
// Check lint status from the separate Lint workflow (lint.yml).
|
|
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
|
|
// checks.listForRef which queries by commit SHA across ALL workflows.
|
|
const ref = context.payload.pull_request?.head?.sha || context.sha;
|
|
core.info(`[health-check] Checking lint for ref=${ref}`);
|
|
const { data } = await github.rest.checks.listForRef({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
ref: ref,
|
|
check_name: 'lint',
|
|
});
|
|
const lintRun = data.check_runs.find(
|
|
cr => cr.app?.slug === 'github-actions'
|
|
);
|
|
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
|
|
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
|
|
core.setFailed('Fail-fast: lint check failed');
|
|
return;
|
|
}
|
|
|
|
// The lint check above is never bypassed; only sibling failures are.
|
|
const { resolveCiLabels } = require(
|
|
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
|
|
);
|
|
const axes = await resolveCiLabels(github, context);
|
|
core.info(`[health-check] PR labels: [${axes.labels.join(', ')}]`);
|
|
if (axes.bypassFailFast) {
|
|
core.info('[health-check] SKIP jobs-failed check: bypass-fail-fast label present');
|
|
return;
|
|
}
|
|
|
|
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
run_id: context.runId,
|
|
per_page: 100,
|
|
});
|
|
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
|
|
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
|
|
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
|
|
|
|
// Find jobs that failed from a real error, not from fail-fast cascade
|
|
const rootCauseFailures = jobs.filter(j => {
|
|
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
|
|
// h20 runners are flaky (dirty GPU state from prior runs); their failures
|
|
// should not cascade fail-fast to other stages. j.name shape from
|
|
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
|
|
// + optional " (<matrix>)". Split off the base job key before exact
|
|
// match so we cover both inline + reusable forms without confusing
|
|
// 'h20' with the 'h200' prefix.
|
|
const baseName = j.name.split(/[ /]/)[0];
|
|
if (baseName === 'base-c-test-8-gpu-h20') {
|
|
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
|
|
return false;
|
|
}
|
|
// multimodal-gen NPU tests should not cascade fail-fast to perf/accuracy stages.
|
|
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
|
|
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
|
|
return false;
|
|
}
|
|
// If the failing step is the health check, it's a cascade -- skip it
|
|
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
|
|
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
|
|
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
|
|
return false;
|
|
}
|
|
return true;
|
|
});
|
|
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
|
|
|
|
if (rootCauseFailures.length > 0) {
|
|
core.setFailed(`Fail-fast: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
|
|
} else {
|
|
core.info('[health-check] PASS: no root cause failures detected');
|
|
}
|