[CI] Split the CI control labels into four axes and resolve them live (#40527)

This commit is contained in:
Liangsheng Yin
2026-09-21 15:37:28 -07:00
committed by GitHub
parent 66f19f5c46
commit a5c2cc517c
22 changed files with 251 additions and 346 deletions
+2 -107
View File
@@ -39,7 +39,7 @@ on:
type: string
default: ''
skip_pr_test_health_check:
description: 'Set to true to skip the PR test health check (fast-fail gate).'
description: 'Set to true to skip the PR test health check (fail-fast gate).'
type: string
default: 'false'
is_nightly_pipeline_job:
@@ -90,112 +90,7 @@ jobs:
run: |
git config --system --add safe.directory ${GITHUB_WORKSPACE}
- name: Check PR test health
uses: actions/github-script@v8
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
with:
github-token: ${{ inputs.github-token || github.token }}
script: |
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
if (context.eventName === 'schedule') {
core.notice('[health-check] SKIP: scheduled run');
return;
}
// Check lint status from the separate Lint workflow (lint.yml).
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: ref,
check_name: 'lint',
});
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
if (labels.includes('bypass-fastfail')) {
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
return;
}
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fast-fail cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.notice('[health-check] PASS: no root cause failures detected');
}
- uses: ./.github/actions/check-pr-test-health
- name: Install dependencies
env:
@@ -43,7 +43,7 @@ on:
default: '{}'
description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
skip_pr_test_health_check:
description: 'Set to true to skip the PR test health check (fast-fail gate).'
description: 'Set to true to skip the PR test health check (fail-fast gate).'
type: string
default: 'false'
is_nightly_pipeline_job:
@@ -101,112 +101,7 @@ jobs:
# checks out the same commit even if the branch advances mid-run.
ref: ${{ inputs.ref || github.sha }}
- name: Check PR test health
uses: actions/github-script@v8
env:
SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
with:
github-token: ${{ inputs.github-token || github.token }}
script: |
core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
if (context.eventName === 'schedule') {
core.notice('[health-check] SKIP: scheduled run');
return;
}
// Check lint status from the separate Lint workflow (lint.yml).
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
ref: ref,
check_name: 'lint',
});
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
if (labels.includes('bypass-fastfail')) {
core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
return;
}
const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
per_page: 100,
});
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fast-fail cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.notice('[health-check] PASS: no root cause failures detected');
}
- uses: ./.github/actions/check-pr-test-health
- name: Check npu info
run: |
+17 -11
View File
@@ -136,15 +136,24 @@ jobs:
- "test/registered/rust/**"
- ".github/workflows/_pr-test-*.yml"
- name: Resolve CI labels
id: ci-labels
if: github.event_name == 'pull_request'
uses: actions/github-script@v8
with:
script: |
const { resolveCiLabels } = require(
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
);
const axes = await resolveCiLabels(github, context);
core.info(`CI labels: [${axes.labels.join(', ')}]`);
core.setOutput('max_concurrency', String(axes.maxConcurrency));
- name: Determine full-parallel mode
id: parallel-mode
run: |
# `full=true` lifts the matrix-fanout throttle so each suite's
# max_parallel = size. Conditions:
# 1. Scheduled cron run.
# 2. run_all_tests run (manual full dispatch / release) -- a full
# run should mirror the cron's parallelism, not just its test set.
# 3. pull_request event with the `high priority` label.
# max_parallel = size.
FULL=false
if [[ "${{ github.event_name }}" == "schedule" ]]; then
FULL=true
@@ -152,9 +161,9 @@ jobs:
elif [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
FULL=true
echo "run_all_tests -> full parallelism"
elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then
elif [[ "${{ steps.ci-labels.outputs.max_concurrency }}" == "true" ]]; then
FULL=true
echo "high priority PR -> full parallelism"
echo "max-concurrency PR -> full parallelism"
fi
echo "full=$FULL" >> "$GITHUB_OUTPUT"
@@ -189,10 +198,7 @@ jobs:
id: partitions
run: |
# Emit a single JSON output `partitions` keyed by suite name with
# {size, arr, max_parallel} fields per suite. Replaces the prior
# full/low max-parallel presets; `--full-parallel` keeps the
# `high priority` PR / scheduled cron escape hatch.
# See scripts/ci/utils/compute_partitions.py.
# {size, arr, max_parallel} fields per suite.
python3 scripts/ci/utils/compute_partitions.py \
--full-parallel ${{ steps.parallel-mode.outputs.full }} \
--partition-model-file /tmp/partition-model.json \
@@ -8,8 +8,8 @@ on:
required: true
type: string
default: 'pr-test.yml pr-test-extra.yml'
include_high_priority:
description: 'Also cancel runs from high-priority PRs'
include_highest_priority:
description: 'Also cancel runs from PRs labelled highest-priority'
required: false
type: boolean
default: false
@@ -36,7 +36,7 @@ jobs:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
REPO: ${{ github.repository }}
WORKFLOWS: ${{ github.event.inputs.workflows }}
INCLUDE_HIGH_PRIORITY: ${{ github.event.inputs.include_high_priority }}
INCLUDE_HIGHEST_PRIORITY: ${{ github.event.inputs.include_highest_priority }}
INCLUDE_RERUN_TEST: ${{ github.event.inputs.include_rerun_test }}
shell: bash
run: |
@@ -49,7 +49,7 @@ jobs:
fi
echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${WORKFLOW_FILES[*]}"
echo "include_high_priority=$INCLUDE_HIGH_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST"
echo "include_highest_priority=$INCLUDE_HIGHEST_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST"
echo ""
# Decide whether to cancel run_id given a PR-lookup endpoint.
@@ -99,12 +99,12 @@ jobs:
return
fi
if echo "$labels" | grep -Fxq "high priority"; then
if [ "$INCLUDE_HIGH_PRIORITY" != "true" ]; then
echo " 🛑 Skipping (high priority label)"
if echo "$labels" | grep -Fxq "highest-priority"; then
if [ "$INCLUDE_HIGHEST_PRIORITY" != "true" ]; then
echo " 🛑 Skipping (highest-priority label)"
return
fi
echo " ⚠️ High priority PR, but include_high_priority is enabled"
echo " ⚠️ highest-priority PR, but include_highest_priority is enabled"
fi
echo " 🚫 Cancelling..."
+1 -1
View File
@@ -38,7 +38,7 @@ jobs:
const QUOTA_IDLE = 7;
const STALE_DAYS = 90;
const KEEP_LABELS = new Set(['high priority', 'keep-open', 'good first issue']);
const KEEP_LABELS = new Set(['highest-priority', 'keep-open', 'good first issue']);
const WIP_MARKER = /^\s*\[?\s*(wip|do[ _-]?not[ _-]?merge|dnm|draft)\s*\]?/i;
// Scheduled runs always act; only manual runs can be dry.
+8 -8
View File
@@ -127,23 +127,23 @@ jobs:
# Get unique PR numbers (exclude NO_PR entries)
pr_numbers=$(cut -d'|' -f1 < "$pr_data_file" | grep -v '^NO_PR$' | sort -u || true)
# Separate high priority and normal PRs
high_priority_prs=()
# Separate highest-priority and normal PRs
highest_priority_prs=()
normal_prs=()
for pr_num in $pr_numbers; do
labels=$(gh pr view "$pr_num" --repo "$REPO" --json labels \
| jq -r '.labels[].name' 2>/dev/null || true)
if echo "$labels" | grep -Fxq "high priority"; then
high_priority_prs+=($pr_num)
if echo "$labels" | grep -Fxq "highest-priority"; then
highest_priority_prs+=($pr_num)
else
normal_prs+=($pr_num)
fi
done
# Combine: high priority first, then normal
sorted_pr_numbers=("${high_priority_prs[@]}" "${normal_prs[@]}")
# Combine: highest-priority first, then normal
sorted_pr_numbers=("${highest_priority_prs[@]}" "${normal_prs[@]}")
pr_count=0
total_running=0
@@ -170,8 +170,8 @@ jobs:
# Add priority indicator
priority_indicator=""
if echo "$pr_labels" | grep -q "high priority"; then
priority_indicator="🔴 [HIGH PRIORITY] "
if echo "$pr_info" | jq -e '[.labels[].name] | index("highest-priority")' >/dev/null; then
priority_indicator="🔴 [HIGHEST PRIORITY] "
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+8 -1
View File
@@ -243,7 +243,14 @@ jobs:
extra-a-test-1-gpu-large-amd,
extra-a-test-2-gpu-large-amd,
]
if: always()
# Same `labeled` guard as call-gate: an unrelated label would otherwise finish
# green with nothing executed, over the real run's result.
if: |
always() &&
(github.event_name != 'pull_request' ||
github.event.action != 'labeled' ||
github.event.label.name == 'run-ci' ||
github.event.label.name == 'run-ci-extra')
runs-on: ubuntu-latest
steps:
- name: Check all dependent job statuses
+16 -6
View File
@@ -187,19 +187,29 @@ jobs:
echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
fi
- name: Resolve CI labels
id: ci-labels
if: github.event_name == 'pull_request'
uses: actions/github-script@v8
with:
script: |
const { resolveCiLabels } = require(
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
);
const axes = await resolveCiLabels(github, context);
core.info(`CI labels: [${axes.labels.join(', ')}]`);
core.setOutput('bypass_fail_fast', String(axes.bypassFailFast));
- name: Set continue-on-error for schedule/full runs
id: set-continue-on-error
env:
# `bypass-fastfail` PR label: also disable within-suite fast-fail
# here. The shared actions/wait-for-jobs already honors the same
# label to skip cross-stage waits.
BYPASS_FASTFAIL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') }}
BYPASS_FAIL_FAST: ${{ steps.ci-labels.outputs.bypass_fail_fast }}
run: |
if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" \
|| "${{ inputs.continue_on_error }}" == "true" \
|| "$BYPASS_FASTFAIL_LABEL" == "true" ]]; then
|| "$BYPASS_FAIL_FAST" == "true" ]]; then
echo "continue_on_error=true" >> $GITHUB_OUTPUT
echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fastfail=$BYPASS_FASTFAIL_LABEL)"
echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fail-fast=$BYPASS_FAIL_FAST)"
else
echo "continue_on_error=false" >> $GITHUB_OUTPUT
echo "Continue-on-error: DISABLED"
+16 -4
View File
@@ -50,13 +50,14 @@ on:
type: boolean
default: false
skip_pr_test_health_check:
description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
description: "Skip PR test health check fail-fast (e.g. for release branch cuts)"
required: false
type: boolean
default: false
concurrency:
group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
# Keys on the PR number: two forks can share a head_ref and would cancel each other.
group: pr-test-extra-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
env:
@@ -202,7 +203,14 @@ jobs:
simulator-test-cpu,
extra-test,
]
if: always()
# Without this guard an unrelated label starts a run that finishes green with
# nothing executed, over the real failure; `skipped` is what pr-states.yml reads.
if: |
always() &&
(github.event_name != 'pull_request' ||
github.event.action != 'labeled' ||
github.event.label.name == 'run-ci' ||
github.event.label.name == 'run-ci-extra')
runs-on: ubuntu-latest
steps:
- name: Check all dependent job statuses
@@ -232,10 +240,14 @@ jobs:
# there (initial completion) and pull_request_target (push / label).
notify-pr-states:
needs: [pr-test-extra-finish]
# Same guard; pr-states.yml subscribes to labeled/unlabeled itself, so nothing is lost.
if: |
always() &&
github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.full_name == github.repository
github.event.pull_request.head.repo.full_name == github.repository &&
(github.event.action != 'labeled' ||
github.event.label.name == 'run-ci' ||
github.event.label.name == 'run-ci-extra')
runs-on: ubuntu-latest
steps:
- name: Dispatch pr-states refresh
+4 -9
View File
@@ -41,17 +41,17 @@ on:
type: boolean
default: false
skip_pr_test_health_check:
description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
description: "Skip PR test health check fail-fast (e.g. for release branch cuts)"
required: false
type: boolean
default: false
concurrency:
# Concurrency group structure: pr-test-{event}-{branch}-{git_ref}
# - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
# (without it, both resolve the branch segment to 'main' and block each other)
# - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name
group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
# - a PR keys on its number: github.head_ref is a bare branch name with no owner,
# so two forks using the same name would share one group and cancel each other
group: pr-test-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
env:
@@ -88,11 +88,6 @@ jobs:
secrets: inherit
# =============================================== Wait Jobs for Sequential PR Execution ====================================================
# These jobs poll GitHub API to wait for previous stages to complete.
# For PR runs: wait jobs run and enforce sequential execution via polling.
# For scheduled runs: wait jobs are skipped, enabling parallel execution for easier retry.
# For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately
# (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel.
wait-for-base-a:
needs: [check-changes, call-gate]