[CI] pr-test: Let stage wait jobs reach their max-wait budget (#44410)

This commit is contained in:
Xinyuan Tong
2026-10-15 13:34:49 -07:00
committed by GitHub
parent aa5c7e887c
commit 877be2974a
3 changed files with 37 additions and 14 deletions
+21 -6
View File
@@ -12,13 +12,17 @@ inputs:
- an object { "prefix": "...", "expected_count": N }: for matrix jobs
required: true
max-wait-minutes:
description: 'Maximum time to wait before timing out'
description: 'Maximum wall-clock time to wait before reporting `timeout`. On GitHub-hosted runners, whose jobs are hard-capped at 6h, keep it under 360 with room for job setup.'
required: false
default: '240'
poll-interval-seconds:
description: 'Seconds between polling attempts'
required: false
default: '60'
fail-on-timeout:
description: 'Fail when max-wait-minutes runs out, unless set to `false`, which reports `timeout` as a warning so later stages proceed.'
required: false
default: 'true'
github-token:
description: 'GitHub token for API calls'
required: false
@@ -40,6 +44,7 @@ runs:
INPUT_JOBS: ${{ inputs.jobs }}
INPUT_MAX_WAIT_MINUTES: ${{ inputs.max-wait-minutes }}
INPUT_POLL_INTERVAL_SECONDS: ${{ inputs.poll-interval-seconds }}
INPUT_FAIL_ON_TIMEOUT: ${{ inputs.fail-on-timeout }}
with:
github-token: ${{ inputs.github-token }}
script: |
@@ -47,7 +52,8 @@ runs:
const jobSpecs = JSON.parse(process.env.INPUT_JOBS);
const maxWaitMinutes = parseInt(process.env.INPUT_MAX_WAIT_MINUTES);
const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS);
const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds;
// Each poll also spends time on API calls, so only a deadline bounds the real wait.
const deadlineMs = Date.now() + maxWaitMinutes * 60 * 1000;
const { resolveCiLabels } = require(
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
@@ -136,7 +142,7 @@ runs:
}
}
for (let attempt = 0; attempt < maxAttempts; attempt++) {
for (let attempt = 0; ; attempt++) {
const { jobs, cached } = await fetchJobs();
let allCompleted = true;
@@ -205,9 +211,18 @@ runs:
return;
}
console.log(`Waiting ${pollIntervalSeconds}s... (attempt ${attempt + 1}/${maxAttempts})`);
await new Promise(resolve => setTimeout(resolve, pollIntervalSeconds * 1000));
const remainingMs = deadlineMs - Date.now();
if (remainingMs <= 0) {
break;
}
const sleepMs = Math.min(pollIntervalSeconds * 1000, remainingMs);
console.log(`Waiting ${Math.ceil(sleepMs / 1000)}s... (attempt ${attempt + 1}, ${Math.ceil(remainingMs / 60000)} min left of ${maxWaitMinutes})`);
await new Promise(resolve => setTimeout(resolve, sleepMs));
}
core.setFailed(`Timeout waiting for ${stageName} jobs`);
core.setOutput('result', 'timeout');
if (process.env.INPUT_FAIL_ON_TIMEOUT !== 'false') {
core.setFailed(`Timeout waiting for ${stageName} jobs`);
} else {
core.warning(`Timeout waiting for ${stageName} jobs; proceeding without the stage gate`);
}
+14 -6
View File
@@ -148,7 +148,12 @@ jobs:
{"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}},
{"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}}
]
max-wait-minutes: '480'
# 10 min under the 6h GitHub-hosted job cap: setup needs under 2 min, the rest is margin;
# timeout-minutes cannot raise the cap, and a wait killed by it ends `cancelled`.
max-wait-minutes: '350'
# pr-test-finish fails the run on any base-b failure, so this gate only saves GPU time;
# a slow base-b lets base-c start rather than costing the base-c run.
fail-on-timeout: 'false'
# =============================================== PR Gate ====================================================
call-gate:
@@ -344,7 +349,8 @@ jobs:
base-b-test:
name: base-b-test-${{ matrix.runner_config }}
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() }}
# A cancelled need is neither failure() nor cancelled(), so check it explicitly.
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' }}
strategy:
fail-fast: false
matrix:
@@ -388,8 +394,10 @@ jobs:
base-c-test:
name: base-c-test-${{ matrix.runner_config }}
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() }}
# wait-for-base-a is listed only to read its result: when it is cancelled,
# wait-for-base-b ends `skipped`, which alone would let this stage through.
needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }}
strategy:
fail-fast: false
matrix:
@@ -431,8 +439,8 @@ jobs:
# the aarch64 build, and neither table should wait on the other's.
base-c-test-aarch64:
name: base-c-test-${{ matrix.runner_config }}
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
if: ${{ !failure() && !cancelled() }}
needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }}
strategy:
fail-fast: false
matrix: