From 877be2974aa151a984e52822c028276c2b1e9303 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:34:49 +0800 Subject: [PATCH] [CI] pr-test: Let stage wait jobs reach their max-wait budget (#44410) --- .agents/skills/ci-workflow-guide/SKILL.md | 4 ++-- .github/actions/wait-for-jobs/action.yml | 27 ++++++++++++++++++----- .github/workflows/pr-test.yml | 20 ++++++++++++----- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.agents/skills/ci-workflow-guide/SKILL.md b/.agents/skills/ci-workflow-guide/SKILL.md index 1aec68cca2..27dcaf522c 100644 --- a/.agents/skills/ci-workflow-guide/SKILL.md +++ b/.agents/skills/ci-workflow-guide/SKILL.md @@ -167,7 +167,7 @@ describes what each one does. 3. If any matched job has `conclusion === 'failure'` → fail immediately (fail-fast) 4. If all matched jobs are completed and count matches `expected_count` → success 5. Otherwise → sleep `poll-interval-seconds` (default: 60s) and retry -6. Timeout after `max-wait-minutes` (240 min for base-a, 480 min for base-b) +6. Timeout after `max-wait-minutes` of wall-clock time (240 min for base-a, 350 min for base-b). Keep budgets under the 6h hard cap on GitHub-hosted runners: a wait killed by the cap ends `cancelled`, not `timeout`, and `timeout-minutes` cannot raise it. The base-b wait sets `fail-on-timeout: 'false'`, so a timeout lets base-c start; `pr-test-finish` fails the run on any base-b failure regardless **Job specs example** (base-b): ```json @@ -378,7 +378,7 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage} |---------|-------------|---------------| | All stage-B/C jobs green but steps skipped | Earlier job failed, `check-pr-test-health` triggered | Find the actual failed job (red X) | | `wait-for-base-b` timeout | `expected_count` doesn't match matrix size | Verify job spec counts match `matrix:` array length | -| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation | +| `pr-test-finish` fails but all jobs green | A job was `cancelled` (counts as failure in finish) | Check concurrency cancellation, and whether a `wait-for-base-*` job hit the 6h hosted-runner cap | | Tests pass locally but fail in CI | Partition assignment, runner GPU type, or `est_time` inaccuracy | Check which partition the test lands in; verify runner label | | Flaky test retried and passed | Retriable failure (accuracy/perf) | Check `[CI Retry]` markers in job logs | | Flaky test NOT retried | Matched non-retriable pattern | Check if error matches `NON_RETRIABLE_PATTERNS` in `ci_utils.py` | diff --git a/.github/actions/wait-for-jobs/action.yml b/.github/actions/wait-for-jobs/action.yml index 846bd9118b..0448f99319 100644 --- a/.github/actions/wait-for-jobs/action.yml +++ b/.github/actions/wait-for-jobs/action.yml @@ -12,13 +12,17 @@ inputs: - an object { "prefix": "...", "expected_count": N }: for matrix jobs required: true max-wait-minutes: - description: 'Maximum time to wait before timing out' + description: 'Maximum wall-clock time to wait before reporting `timeout`. On GitHub-hosted runners, whose jobs are hard-capped at 6h, keep it under 360 with room for job setup.' required: false default: '240' poll-interval-seconds: description: 'Seconds between polling attempts' required: false default: '60' + fail-on-timeout: + description: 'Fail when max-wait-minutes runs out, unless set to `false`, which reports `timeout` as a warning so later stages proceed.' + required: false + default: 'true' github-token: description: 'GitHub token for API calls' required: false @@ -40,6 +44,7 @@ runs: INPUT_JOBS: ${{ inputs.jobs }} INPUT_MAX_WAIT_MINUTES: ${{ inputs.max-wait-minutes }} INPUT_POLL_INTERVAL_SECONDS: ${{ inputs.poll-interval-seconds }} + INPUT_FAIL_ON_TIMEOUT: ${{ inputs.fail-on-timeout }} with: github-token: ${{ inputs.github-token }} script: | @@ -47,7 +52,8 @@ runs: const jobSpecs = JSON.parse(process.env.INPUT_JOBS); const maxWaitMinutes = parseInt(process.env.INPUT_MAX_WAIT_MINUTES); const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS); - const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds; + // Each poll also spends time on API calls, so only a deadline bounds the real wait. + const deadlineMs = Date.now() + maxWaitMinutes * 60 * 1000; const { resolveCiLabels } = require( `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs` @@ -136,7 +142,7 @@ runs: } } - for (let attempt = 0; attempt < maxAttempts; attempt++) { + for (let attempt = 0; ; attempt++) { const { jobs, cached } = await fetchJobs(); let allCompleted = true; @@ -205,9 +211,18 @@ runs: return; } - console.log(`Waiting ${pollIntervalSeconds}s... (attempt ${attempt + 1}/${maxAttempts})`); - await new Promise(resolve => setTimeout(resolve, pollIntervalSeconds * 1000)); + const remainingMs = deadlineMs - Date.now(); + if (remainingMs <= 0) { + break; + } + const sleepMs = Math.min(pollIntervalSeconds * 1000, remainingMs); + console.log(`Waiting ${Math.ceil(sleepMs / 1000)}s... (attempt ${attempt + 1}, ${Math.ceil(remainingMs / 60000)} min left of ${maxWaitMinutes})`); + await new Promise(resolve => setTimeout(resolve, sleepMs)); } - core.setFailed(`Timeout waiting for ${stageName} jobs`); core.setOutput('result', 'timeout'); + if (process.env.INPUT_FAIL_ON_TIMEOUT !== 'false') { + core.setFailed(`Timeout waiting for ${stageName} jobs`); + } else { + core.warning(`Timeout waiting for ${stageName} jobs; proceeding without the stage gate`); + } diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index ddf58629bc..d72065c3a8 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -148,7 +148,12 @@ jobs: {"prefix": "base-b-test-2-gpu-large", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-2-gpu-large'].size }}}, {"prefix": "base-b-test-4-gpu-b200", "expected_count": ${{ fromJson(needs.check-changes.outputs.partitions)['base-b-test-4-gpu-b200'].size }}} ] - max-wait-minutes: '480' + # 10 min under the 6h GitHub-hosted job cap: setup needs under 2 min, the rest is margin; + # timeout-minutes cannot raise the cap, and a wait killed by it ends `cancelled`. + max-wait-minutes: '350' + # pr-test-finish fails the run on any base-b failure, so this gate only saves GPU time; + # a slow base-b lets base-c start rather than costing the base-c run. + fail-on-timeout: 'false' # =============================================== PR Gate ==================================================== call-gate: @@ -344,7 +349,8 @@ jobs: base-b-test: name: base-b-test-${{ matrix.runner_config }} needs: [check-changes, call-gate, wait-for-base-a, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} + # A cancelled need is neither failure() nor cancelled(), so check it explicitly. + if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' }} strategy: fail-fast: false matrix: @@ -388,8 +394,10 @@ jobs: base-c-test: name: base-c-test-${{ matrix.runner_config }} - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] - if: ${{ !failure() && !cancelled() }} + # wait-for-base-a is listed only to read its result: when it is cancelled, + # wait-for-base-b ends `skipped`, which alone would let this stage through. + needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build] + if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }} strategy: fail-fast: false matrix: @@ -431,8 +439,8 @@ jobs: # the aarch64 build, and neither table should wait on the other's. base-c-test-aarch64: name: base-c-test-${{ matrix.runner_config }} - needs: [check-changes, call-gate, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64] - if: ${{ !failure() && !cancelled() }} + needs: [check-changes, call-gate, wait-for-base-a, wait-for-base-b, sgl-kernel-build-wheels, rust-ext-build-aarch64] + if: ${{ !failure() && !cancelled() && needs.wait-for-base-a.result != 'cancelled' && needs.wait-for-base-b.result != 'cancelled' }} strategy: fail-fast: false matrix: