minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Split the CI control labels into four axes and resolve them live (#40527)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-09-21 15:37:28 -07:00
committed by GitHub
parent 66f19f5c46
commit a5c2cc517c
22 changed files with 251 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/check-pr-test-health/action.test.cjs
+10 -7
View File
@@ -6,8 +6,10 @@ const { test } = require('node:test');
const yaml = fs.readFileSync(path.join(__dirname, 'action.yml'), 'utf8');
const script = yaml.split(' script: |\n')[1]
.split('\n').map(line => line.replace(/^ /, '')).join('\n');
// `require` and GITHUB_WORKSPACE mirror what actions/github-script injects.
const WORKSPACE = path.join(__dirname, '..', '..', '..');
const run = new (Object.getPrototypeOf(async function () {}).constructor)(
'github', 'context', 'core', 'process', script,
'github', 'context', 'core', 'process', 'require', script,
);
async function check({ snapshot = [], live = [], lint = 'success', event = 'pull_request' } = {}) {
@@ -41,29 +43,30 @@ async function check({ snapshot = [], live = [], lint = 'success', event = 'pull
payload: event === 'pull_request'
? { pull_request: { number: 42, head: { sha: 'head' }, labels: labels(snapshot) } }
: {},
}, { info: () => {}, setFailed: message => failures.push(message) }, { env: {} });
}, { info: () => {}, setFailed: message => failures.push(message) },
{ env: { GITHUB_WORKSPACE: WORKSPACE } }, require);
return { failures, labelReads, jobReads };
}
test('a label added after the event bypasses sibling failures on rerun', async () => {
assert.deepEqual(await check({ live: ['bypass-fastfail'] }),
assert.deepEqual(await check({ live: ['bypass-fail-fast'] }),
{ failures: [], labelReads: 1, jobReads: 0 });
});
test('a removed label does not continue bypassing sibling failures', async () => {
const result = await check({ snapshot: ['bypass-fastfail'] });
const result = await check({ snapshot: ['bypass-fail-fast'] });
assert.equal(result.labelReads, 1);
assert.equal(result.jobReads, 1);
assert.match(result.failures[0], /root cause job\(s\): model-test/);
});
test('bypass never skips a failed lint check', async () => {
assert.deepEqual(await check({ live: ['bypass-fastfail'], lint: 'failure' }),
{ failures: ['Fast-fail: lint check failed'], labelReads: 0, jobReads: 0 });
assert.deepEqual(await check({ live: ['bypass-fail-fast'], lint: 'failure' }),
{ failures: ['Fail-fast: lint check failed'], labelReads: 0, jobReads: 0 });
});
test('non-PR events retain associated-PR label lookup', async () => {
assert.deepEqual(await check({ event: 'workflow_dispatch', live: ['bypass-fastfail'] }),
assert.deepEqual(await check({ event: 'workflow_dispatch', live: ['bypass-fail-fast'] }),
{ failures: [], labelReads: 1, jobReads: 0 });
});
.github/actions/check-pr-test-health/action.yml
+36 -32
View File
@@ -1,5 +1,5 @@
name: Check PR Test Health
description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fastfail` label.
description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fail-fast` label (or `highest-priority`, which implies it).
inputs:
github-token:
@@ -17,15 +17,17 @@ runs:
with:
github-token: ${{ inputs.github-token }}
script: |
core.info(`[health-check] START -- event=${context.eventName}, runId=${context.runId}`);
// Skip when explicitly requested via env var (e.g. release branch cut)
if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
core.info('Skipping health check (SKIP_PR_TEST_HEALTH_CHECK=true)');
core.info('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
return;
}
// Skip for scheduled runs — they should collect all failures, not fast-fail
// Skip for scheduled runs -- they should collect all failures, not fail-fast
if (context.eventName === 'schedule') {
core.info('Skipping health check for scheduled run');
core.info('[health-check] SKIP: scheduled run');
return;
}
@@ -33,6 +35,7 @@ runs:
// listJobsForWorkflowRun only sees jobs within the SAME run, so we use
// checks.listForRef which queries by commit SHA across ALL workflows.
const ref = context.payload.pull_request?.head?.sha || context.sha;
core.info(`[health-check] Checking lint for ref=${ref}`);
const { data } = await github.rest.checks.listForRef({
owner: context.repo.owner,
repo: context.repo.repo,
@@ -42,34 +45,20 @@ runs:
const lintRun = data.check_runs.find(
cr => cr.app?.slug === 'github-actions'
);
core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
core.setFailed('Fast-fail: lint check failed');
core.setFailed('Fail-fast: lint check failed');
return;
}
// Skip the jobs-failed check when the PR carries the bypass-fastfail label.
// Lint check above still runs.
let labels = [];
if (context.payload.pull_request?.number) {
// reruns retain the original event payload, including stale labels
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
});
labels = pr.labels.map(l => l.name);
} else {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
}
if (labels.includes('bypass-fastfail')) {
core.info('Skipping jobs-failed check (bypass-fastfail label present)');
// The lint check above is never bypassed; only sibling failures are.
const { resolveCiLabels } = require(
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
);
const axes = await resolveCiLabels(github, context);
core.info(`[health-check] PR labels: [${axes.labels.join(', ')}]`);
if (axes.bypassFailFast) {
core.info('[health-check] SKIP jobs-failed check: bypass-fail-fast label present');
return;
}
@@ -79,26 +68,41 @@ runs:
run_id: context.runId,
per_page: 100,
});
// Find jobs that failed from a real error, not from fast-fail cascade
core.info(`[health-check] Total jobs in run: ${jobs.length}`);
const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
// Find jobs that failed from a real error, not from fail-fast cascade
const rootCauseFailures = jobs.filter(j => {
if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
// h20 runners are flaky (dirty GPU state from prior runs); their failures
// should not cascade fast-fail to other stages. j.name shape from
// should not cascade fail-fast to other stages. j.name shape from
// listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
// + optional " (<matrix>)". Split off the base job key before exact
// match so we cover both inline + reusable forms without confusing
// 'h20' with the 'h200' prefix.
const baseName = j.name.split(/[ /]/)[0];
if (baseName === 'base-c-test-8-gpu-h20') {
core.info(`[health-check] Filtered out h20 job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade — skip it
// multimodal-gen NPU tests should not cascade fail-fast to perf/accuracy stages.
if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
return false;
}
// If the failing step is the health check, it's a cascade -- skip it
const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
return false;
}
return true;
});
core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
if (rootCauseFailures.length > 0) {
core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
core.setFailed(`Fail-fast: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
} else {
core.info('[health-check] PASS: no root cause failures detected');
}
.github/actions/wait-for-jobs/action.yml
+7 -23
View File
@@ -1,5 +1,5 @@
name: Wait for Jobs
description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `bypass-fastfail` label, letting downstream stages dispatch in parallel (same effect as scheduled runs).
description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `parallel-stages` label (or `highest-priority`, which implies it), letting downstream stages dispatch in parallel (same effect as scheduled runs).
inputs:
stage-name:
@@ -49,28 +49,12 @@ runs:
const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS);
const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds;
// bypass-fastfail label opts the PR out of stage-to-stage waiting,
// letting all stages dispatch in parallel like scheduled runs do.
let labels = [];
if (context.payload.pull_request?.labels) {
labels = context.payload.pull_request.labels.map(l => l.name);
} else {
const ref = context.payload.pull_request?.head?.sha || context.sha;
try {
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: ref,
});
if (prs.length > 0) {
labels = prs[0].labels.map(l => l.name);
}
} catch (e) {
console.log(`Could not fetch PR labels for ${ref}: ${e.message}`);
}
}
if (labels.includes('bypass-fastfail')) {
console.log(`Skipping ${stageName} wait (bypass-fastfail label present)`);
const { resolveCiLabels } = require(
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
);
const { parallelStages } = await resolveCiLabels(github, context);
if (parallelStages) {
console.log(`Skipping ${stageName} wait (parallel-stages label present)`);
core.setOutput('result', 'success');
return;
}