From 3937a07b177274da508cb33391fff50075f69347 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Tue, 24 Mar 2026 16:58:56 -0700 Subject: [PATCH] [CI] Add cross-job fast-fail health check (Layer 3) (#21341) --- .github/actions/check-stage-health/action.yml | 42 +++++++++++++++++++ .github/workflows/pr-test-jit-kernel.yml | 6 +++ .github/workflows/pr-test-multimodal-gen.yml | 9 ++++ .github/workflows/pr-test-sgl-kernel.yml | 12 ++++++ .github/workflows/pr-test.yml | 36 ++++++++++++++++ 5 files changed, 105 insertions(+) create mode 100644 .github/actions/check-stage-health/action.yml diff --git a/.github/actions/check-stage-health/action.yml b/.github/actions/check-stage-health/action.yml new file mode 100644 index 000000000..facec3b37 --- /dev/null +++ b/.github/actions/check-stage-health/action.yml @@ -0,0 +1,42 @@ +name: Check Stage Health +description: Fail fast if any job in the current workflow run has already failed. Auto-skips for scheduled runs. + +inputs: + github-token: + description: 'GitHub token for API calls' + required: false + default: ${{ github.token }} + +runs: + using: composite + steps: + - name: Check stage health + uses: actions/github-script@v7 + with: + github-token: ${{ inputs.github-token }} + script: | + // Skip for scheduled runs — they should collect all failures, not fast-fail + if (context.eventName === 'schedule') { + core.info('Skipping health check for scheduled run'); + return; + } + + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: 100, + }); + // Find jobs that failed from a real error, not from fast-fail cascade + const rootCauseFailures = jobs.filter(j => { + if (j.status !== 'completed' || j.conclusion !== 'failure') return false; + // If the failing step is the health check, it's a cascade — skip it + const failedStep = (j.steps || []).find(s => s.conclusion === 'failure'); + if (failedStep && (failedStep.name.includes('check-stage-health') || failedStep.name.includes('Check stage health'))) { + return false; + } + return true; + }); + if (rootCauseFailures.length > 0) { + core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`); + } diff --git a/.github/workflows/pr-test-jit-kernel.yml b/.github/workflows/pr-test-jit-kernel.yml index 6e5627857..4ee64e4f4 100644 --- a/.github/workflows/pr-test-jit-kernel.yml +++ b/.github/workflows/pr-test-jit-kernel.yml @@ -44,6 +44,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -71,6 +74,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index 4bacb675e..5010134f2 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -62,6 +62,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -116,6 +119,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -167,6 +173,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} diff --git a/.github/workflows/pr-test-sgl-kernel.yml b/.github/workflows/pr-test-sgl-kernel.yml index 1aadebb50..6c9d9652a 100644 --- a/.github/workflows/pr-test-sgl-kernel.yml +++ b/.github/workflows/pr-test-sgl-kernel.yml @@ -34,6 +34,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -69,6 +72,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -104,6 +110,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -151,6 +160,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index cdaa5d6a9..30447c4f5 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -561,6 +561,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -614,6 +617,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -668,6 +674,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -729,6 +738,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -783,6 +795,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -839,6 +854,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -923,6 +941,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -977,6 +998,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -1047,6 +1071,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -1097,6 +1124,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -1157,6 +1187,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }} @@ -1223,6 +1256,9 @@ jobs: with: ref: ${{ inputs.pr_head_sha || inputs.git_ref || github.sha }} + - uses: ./.github/actions/check-stage-health + id: stage-health + - uses: ./.github/actions/check-maintenance with: github-token: ${{ github.token }}