[CI] pr-test: Let stage wait jobs reach their max-wait budget (#44410)

This commit is contained in:
Xinyuan Tong
2026-10-15 13:34:49 -07:00
committed by GitHub
parent aa5c7e887c
commit 877be2974a
3 changed files with 37 additions and 14 deletions
+2 -2
View File
@@ -167,7 +167,7 @@ describes what each one does.
3. If any matched job has `conclusion === 'failure'` → fail immediately (fail-fast)
4. If all matched jobs are completed and count matches `expected_count` → success
5. Otherwise → sleep `poll-interval-seconds` (default: 60s) and retry
6. Timeout after `max-wait-minutes` (240 min for base-a, 480 min for base-b)
6. Timeout after `max-wait-minutes` of wall-clock time (240 min for base-a, 350 min for base-b). Keep budgets under the 6h hard cap on GitHub-hosted runners: a wait killed by the cap ends `cancelled`, not `timeout`, and `timeout-minutes` cannot raise it. The base-b wait sets `fail-on-timeout: 'false'`, so a timeout lets base-c start; `pr-test-finish` fails the run on any base-b failure regardless
**Job specs example** (base-b):
```json
@@ -378,7 +378,7 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
|---------|-------------|---------------|
| All stage-B/C jobs green but steps skipped | Earlier job failed, `check-pr-test-health` triggered | Find the actual failed job (red X) |
| `wait-for-base-b` timeout | `expected_count` doesn't match matrix size | Verify job spec counts match `matrix:` array length |
| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation |
| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation, and whether a `wait-for-base-*` job hit the 6h hosted-runner cap |
| Tests pass locally but fail in CI | Partition assignment, runner GPU type, or `est_time` inaccuracy | Check which partition the test lands in; verify runner label |
| Flaky test retried and passed | Retriable failure (accuracy/perf) | Check `[CI Retry]` markers in job logs |
| Flaky test NOT retried | Matched non-retriable pattern | Check if error matches `NON_RETRIABLE_PATTERNS` in `ci_utils.py` |
+21 -6
View File
@@ -12,13 +12,17 @@ inputs:
- an object { "prefix": "...", "expected_count": N }: for matrix jobs
required: true
max-wait-minutes:
description: 'Maximum time to wait before timing out'
description: 'Maximum wall-clock time to wait before reporting `timeout`. On GitHub-hosted runners, whose jobs are hard-capped at 6h, keep it under 360 with room for job setup.'
required: false
default: '240'
poll-interval-seconds:
description: 'Seconds between polling attempts'
required: false
default: '60'
fail-on-timeout:
description: 'Fail when max-wait-minutes runs out, unless set to `false`, which reports `timeout` as a warning so later stages proceed.'
required: false
default: 'true'
github-token:
description: 'GitHub token for API calls'
required: false
@@ -40,6 +44,7 @@ runs:
INPUT_JOBS: ${{ inputs.jobs }}
INPUT_MAX_WAIT_MINUTES: ${{ inputs.max-wait-minutes }}
INPUT_POLL_INTERVAL_SECONDS: ${{ inputs.poll-interval-seconds }}
INPUT_FAIL_ON_TIMEOUT: ${{ inputs.fail-on-timeout }}
with:
github-token: ${{ inputs.github-token }}
script: |
@@ -47,7 +52,8 @@ runs:
const jobSpecs = JSON.parse(process.env.INPUT_JOBS);
const maxWaitMinutes = parseInt(process.env.INPUT_MAX_WAIT_MINUTES);
const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS);
const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds;
// Each poll also spends time on API calls, so only a deadline bounds the real wait.
const deadlineMs = Date.now() + maxWaitMinutes * 60 * 1000;
const { resolveCiLabels } = require(
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
@@ -136,7 +142,7 @@ runs:
}
}
for (let attempt = 0; attempt < maxAttempts; attempt++) {
for (let attempt = 0; ; attempt++) {
const { jobs, cached } = await fetchJobs();
let allCompleted = true;
@@ -205,9 +211,18 @@ runs:
return;
}
console.log(`Waiting ${pollIntervalSeconds}s... (attempt ${attempt + 1}/${maxAttempts})`);
await new Promise(resolve => setTimeout(resolve, pollIntervalSeconds * 1000));
const remainingMs = deadlineMs - Date.now();
if (remainingMs <= 0) {
break;
}
const sleepMs = Math.min(pollIntervalSeconds * 1000, remainingMs);
console.log(`Waiting ${Math.ceil(sleepMs / 1000)}s... (attempt ${attempt + 1}, ${Math.ceil(remainingMs / 60000)} min left of ${maxWaitMinutes})`);
await new Promise(resolve => setTimeout(resolve, sleepMs));
}
core.setFailed(`Timeout waiting for ${stageName} jobs`);
core.setOutput('result', 'timeout');
if (process.env.INPUT_FAIL_ON_TIMEOUT !== 'false') {
core.setFailed(`Timeout waiting for ${stageName} jobs`);
} else {
core.warning(`Timeout waiting for ${stageName} jobs; proceeding without the stage gate`);
}
+14 -6
View File
@@ -148,7 +148,12 @@ jobs:
{"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}},
{"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}}
]
max-wait-minutes: '480'
# 10 min under the 6h GitHub-hosted job cap: setup needs under 2 min, the rest is margin;
# timeout-minutes cannot raise the cap, and a wait killed by it ends `cancelled`.
max-wait-minutes: '350'
# pr-test-finish fails the run on any base-b failure, so this gate only saves GPU time;
# a slow base-b lets base-c start rather than costing the base-c run.
fail-on-timeout: 'false'
# =============================================== PR Gate ====================================================
call-gate:
@@ -344,7 +349,8 @@ jobs:
base-b-test:
name: base-b-test-${{ matrix.runner_config }}
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() }}
# A cancelled need is neither failure() nor cancelled(), so check it explicitly.
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' }}
strategy:
fail-fast: false
matrix:
@@ -388,8 +394,10 @@ jobs:
base-c-test:
name: base-c-test-${{ matrix.runner_config }}
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() }}
# wait-for-base-a is listed only to read its result: when it is cancelled,
# wait-for-base-b ends `skipped`, which alone would let this stage through.
needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }}
strategy:
fail-fast: false
matrix:
@@ -431,8 +439,8 @@ jobs:
# the aarch64 build, and neither table should wait on the other's.
base-c-test-aarch64:
name: base-c-test-${{ matrix.runner_config }}
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
if: ${{ !failure() && !cancelled() }}
needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }}
strategy:
fail-fast: false
matrix: