[CI] pr-test: Let stage wait jobs reach their max-wait budget (#44410)
This commit is contained in:
@@ -167,7 +167,7 @@ describes what each one does.
|
||||
3. If any matched job has `conclusion === 'failure'` → fail immediately (fail-fast)
|
||||
4. If all matched jobs are completed and count matches `expected_count` → success
|
||||
5. Otherwise → sleep `poll-interval-seconds` (default: 60s) and retry
|
||||
6. Timeout after `max-wait-minutes` (240 min for base-a, 480 min for base-b)
|
||||
6. Timeout after `max-wait-minutes` of wall-clock time (240 min for base-a, 350 min for base-b). Keep budgets under the 6h hard cap on GitHub-hosted runners: a wait killed by the cap ends `cancelled`, not `timeout`, and `timeout-minutes` cannot raise it. The base-b wait sets `fail-on-timeout: 'false'`, so a timeout lets base-c start; `pr-test-finish` fails the run on any base-b failure regardless
|
||||
|
||||
**Job specs example** (base-b):
|
||||
```json
|
||||
@@ -378,7 +378,7 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
|
||||
|---------|-------------|---------------|
|
||||
| All stage-B/C jobs green but steps skipped | Earlier job failed, `check-pr-test-health` triggered | Find the actual failed job (red X) |
|
||||
| `wait-for-base-b` timeout | `expected_count` doesn't match matrix size | Verify job spec counts match `matrix:` array length |
|
||||
| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation |
|
||||
| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation, and whether a `wait-for-base-*` job hit the 6h hosted-runner cap |
|
||||
| Tests pass locally but fail in CI | Partition assignment, runner GPU type, or `est_time` inaccuracy | Check which partition the test lands in; verify runner label |
|
||||
| Flaky test retried and passed | Retriable failure (accuracy/perf) | Check `[CI Retry]` markers in job logs |
|
||||
| Flaky test NOT retried | Matched non-retriable pattern | Check if error matches `NON_RETRIABLE_PATTERNS` in `ci_utils.py` |
|
||||
|
||||
@@ -12,13 +12,17 @@ inputs:
|
||||
- an object { "prefix": "...", "expected_count": N }: for matrix jobs
|
||||
required: true
|
||||
max-wait-minutes:
|
||||
description: 'Maximum time to wait before timing out'
|
||||
description: 'Maximum wall-clock time to wait before reporting `timeout`. On GitHub-hosted runners, whose jobs are hard-capped at 6h, keep it under 360 with room for job setup.'
|
||||
required: false
|
||||
default: '240'
|
||||
poll-interval-seconds:
|
||||
description: 'Seconds between polling attempts'
|
||||
required: false
|
||||
default: '60'
|
||||
fail-on-timeout:
|
||||
description: 'Fail when max-wait-minutes runs out, unless set to `false`, which reports `timeout` as a warning so later stages proceed.'
|
||||
required: false
|
||||
default: 'true'
|
||||
github-token:
|
||||
description: 'GitHub token for API calls'
|
||||
required: false
|
||||
@@ -40,6 +44,7 @@ runs:
|
||||
INPUT_JOBS: ${{ inputs.jobs }}
|
||||
INPUT_MAX_WAIT_MINUTES: ${{ inputs.max-wait-minutes }}
|
||||
INPUT_POLL_INTERVAL_SECONDS: ${{ inputs.poll-interval-seconds }}
|
||||
INPUT_FAIL_ON_TIMEOUT: ${{ inputs.fail-on-timeout }}
|
||||
with:
|
||||
github-token: ${{ inputs.github-token }}
|
||||
script: |
|
||||
@@ -47,7 +52,8 @@ runs:
|
||||
const jobSpecs = JSON.parse(process.env.INPUT_JOBS);
|
||||
const maxWaitMinutes = parseInt(process.env.INPUT_MAX_WAIT_MINUTES);
|
||||
const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS);
|
||||
const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds;
|
||||
// Each poll also spends time on API calls, so only a deadline bounds the real wait.
|
||||
const deadlineMs = Date.now() + maxWaitMinutes * 60 * 1000;
|
||||
|
||||
const { resolveCiLabels } = require(
|
||||
`${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
|
||||
@@ -136,7 +142,7 @@ runs:
|
||||
}
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
const { jobs, cached } = await fetchJobs();
|
||||
|
||||
let allCompleted = true;
|
||||
@@ -205,9 +211,18 @@ runs:
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Waiting ${pollIntervalSeconds}s... (attempt ${attempt + 1}/${maxAttempts})`);
|
||||
await new Promise(resolve => setTimeout(resolve, pollIntervalSeconds * 1000));
|
||||
const remainingMs = deadlineMs - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
break;
|
||||
}
|
||||
const sleepMs = Math.min(pollIntervalSeconds * 1000, remainingMs);
|
||||
console.log(`Waiting ${Math.ceil(sleepMs / 1000)}s... (attempt ${attempt + 1}, ${Math.ceil(remainingMs / 60000)} min left of ${maxWaitMinutes})`);
|
||||
await new Promise(resolve => setTimeout(resolve, sleepMs));
|
||||
}
|
||||
|
||||
core.setFailed(`Timeout waiting for ${stageName} jobs`);
|
||||
core.setOutput('result', 'timeout');
|
||||
if (process.env.INPUT_FAIL_ON_TIMEOUT !== 'false') {
|
||||
core.setFailed(`Timeout waiting for ${stageName} jobs`);
|
||||
} else {
|
||||
core.warning(`Timeout waiting for ${stageName} jobs; proceeding without the stage gate`);
|
||||
}
|
||||
|
||||
@@ -148,7 +148,12 @@ jobs:
|
||||
{"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}},
|
||||
{"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}}
|
||||
]
|
||||
max-wait-minutes: '480'
|
||||
# 10 min under the 6h GitHub-hosted job cap: setup needs under 2 min, the rest is margin;
|
||||
# timeout-minutes cannot raise the cap, and a wait killed by it ends `cancelled`.
|
||||
max-wait-minutes: '350'
|
||||
# pr-test-finish fails the run on any base-b failure, so this gate only saves GPU time;
|
||||
# a slow base-b lets base-c start rather than costing the base-c run.
|
||||
fail-on-timeout: 'false'
|
||||
|
||||
# =============================================== PR Gate ====================================================
|
||||
call-gate:
|
||||
@@ -344,7 +349,8 @@ jobs:
|
||||
base-b-test:
|
||||
name: base-b-test-${{ matrix.runner_config }}
|
||||
needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
# A cancelled need is neither failure() nor cancelled(), so check it explicitly.
|
||||
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -388,8 +394,10 @@ jobs:
|
||||
|
||||
base-c-test:
|
||||
name: base-c-test-${{ matrix.runner_config }}
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
# wait-for-base-a is listed only to read its result: when it is cancelled,
|
||||
# wait-for-base-b ends `skipped`, which alone would let this stage through.
|
||||
needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build]
|
||||
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
@@ -431,8 +439,8 @@ jobs:
|
||||
# the aarch64 build, and neither table should wait on the other's.
|
||||
base-c-test-aarch64:
|
||||
name: base-c-test-${{ matrix.runner_config }}
|
||||
needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64]
|
||||
if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
||||
Reference in New Issue
Block a user