From 3ec22948c122f99f742b1c1c0921893b6c7de958 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:15:54 -0700 Subject: [PATCH] [CI] /rerun-failed-ci: rerun cancelled runs and target the newest run per workflow (#34057) --- .../developer_guide/contribution_guide.mdx | 2 +- .../development/contribution_guide.mdx | 2 +- scripts/ci/utils/slash_command_handler.py | 49 ++++++++++++++++--- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/docs/docs/developer_guide/contribution_guide.mdx b/docs/docs/developer_guide/contribution_guide.mdx index bd14f2540..f320cbf68 100644 --- a/docs/docs/developer_guide/contribution_guide.mdx +++ b/docs/docs/developer_guide/contribution_guide.mdx @@ -120,7 +120,7 @@ Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: - `/tag-run-ci-label`: Adds the "run-ci" label. Only **future** commits trigger CI; the current commit is unaffected. Add the `extra` argument (`/tag-run-ci-label extra`) to additionally apply the "run-ci-extra" label, opting the PR into the extra test workflow (`pr-test-extra.yml`). -- `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed or skipped**. +- `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed, skipped, cancelled, or timed out**. - `/tag-and-rerun-ci`: Runs both. Use this on a fresh PR to kick off CI on the current commit — `/tag-run-ci-label` alone won't. Accepts the same `extra` argument (`/tag-and-rerun-ci extra`). - `/rerun-test [ ...]`: Reruns one or more specific tests directly. A spec may select a file, class, or method using `::[.]`. Multiple specs and file globs are supported. Examples: `/rerun-test test_srt_endpoint.py`, `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`, `/rerun-test test_a.py test_b.py`, or `/rerun-test test_*backend*.py`. - `/rerun-group [ ...]`: Expands one or more registered test groups (for example, `/rerun-group hicache`) and dispatches their tests through the same selective-rerun workflow. diff --git a/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx b/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx index cdfe91aad..92d7d5397 100644 --- a/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx +++ b/docs/docs/hardware-platforms/ascend-npus/development/contribution_guide.mdx @@ -144,7 +144,7 @@ Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: - `/tag-run-ci-label`: Adds the "run-ci" label. Only **future** commits trigger CI; the current commit is unaffected. Add the `extra` argument (`/tag-run-ci-label extra`) to additionally apply the "run-ci-extra" label, opting the PR into the extra test workflow (`pr-test-extra.yml`). -- `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed, flaky, or skipped**. +- `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed, flaky, skipped, cancelled, or timed out**. - `/tag-and-rerun-ci`: Runs both. Use this on a fresh PR to kick off CI on the current commit — `/tag-run-ci-label` alone won't. Accepts the same `extra` argument (`/tag-and-rerun-ci extra`). - `/rerun-test [ ...]`: Reruns one or more specific tests directly, bypassing stage boundaries. Each `` is pytest-style `::[.]` (the `::TestClass` and `.` parts are optional). The handler resolves each spec, groups specs by their registered runner-label, and dispatches one [Rerun Test workflow](https://github.com/sgl-project/sglang/actions/workflows/rerun-test.yml) per group. Examples: `/rerun-test test_srt_endpoint.py`, `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`, `/rerun-test test_a.py test_b.py` (multiple at once). diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index c2ff64cd5..1bd0e7717 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -351,17 +351,36 @@ def handle_tag_run_ci( return True +def _latest_run_per_workflow(runs): + """ + Collapse a head_sha's workflow runs to the newest run per workflow. + + GitHub can have several runs of the *same* workflow at one commit — a + `synchronize` run superseded by a `labeled` run, or a run cancelled by + `cancel-in-progress` (pr-test.yml sets it) while its replacement is + already in flight. Only the newest one reflects current state; rerunning + the older ones spawns duplicates that fight the live run for runners. + """ + latest = {} + for run in runs: + current = latest.get(run.workflow_id) + if current is None or run.id > current.id: + latest[run.workflow_id] = run + return list(latest.values()) + + def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=True): """ Handles the /rerun-failed-ci command. - Reruns workflows with 'failure' or 'skipped' conclusions. + Reruns workflows that ended in 'failure', 'skipped', 'cancelled' or + 'timed_out', restarting only the jobs that didn't succeed. Returns True if action was taken, False otherwise. """ if not user_perms.get("can_rerun_failed_ci", False): print("Permission denied: can_rerun_failed_ci is false.") return False - print("Permission granted. Triggering rerun of failed or skipped workflows.") + print("Permission granted. Triggering rerun of unsuccessful workflows.") # Check if PR has sgl-kernel changes - if so, we may need full reruns # to ensure sgl-kernel-build-wheels runs and produces fresh artifacts. @@ -407,7 +426,7 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr f"Failed to check kernel wheel status: {e} - falling back to full rerun" ) - # Rerun workflows with conclusion=failure or conclusion=skipped. + # Rerun workflows that ended in failure, skipped, cancelled or timed_out. # # - failure: use rerun_failed_jobs() which reruns failed jobs *and their # dependent jobs* (GitHub API). Fast-fail cascades call @@ -422,17 +441,27 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr # label-gated workflow, add the missing label (the `labeled` event # dispatches a fresh run with the current label set); this function # cannot recover those by rerun alone. + # - cancelled / timed_out: rerun_failed_jobs() as well. GitHub restarts + # every job that didn't succeed — cancelled ones included — plus their + # dependents, and carries the passing jobs over untouched. A full + # rerun here would re-execute dozens of already-green GPU jobs to + # recover one cancelled partition. + # A run cancelled before any job failed has nothing for the endpoint to + # target, so fall back to run.rerun() when it rejects the request. # - kernel wheel escape: if the PR touches sgl-kernel and not all wheel # builds are success yet, full-rerun failure runs too — Build Wheel # lives in pr-test-sgl-kernel.yml, consumers in pr-test.yml, and # rerun_failed_jobs() is scoped to a single workflow run. - runs = gh_repo.get_workflow_runs(head_sha=head_sha) + runs = _latest_run_per_workflow(gh_repo.get_workflow_runs(head_sha=head_sha)) rerun_count = 0 for run in runs: if run.status != "completed": + # A newer attempt is still in flight - nothing to recover. continue - if run.conclusion not in ("failure", "skipped"): + if run.conclusion not in ("failure", "skipped", "cancelled", "timed_out"): + # action_required (fork PR awaiting approval) is deliberately left + # out: it needs an approval, not a rerun. continue print(f"Processing {run.conclusion} workflow: {run.name} (ID: {run.id})") @@ -443,8 +472,12 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr print(" Full rerun") run.rerun() else: - print(" rerun_failed_jobs") - run.rerun_failed_jobs() + try: + print(" rerun_failed_jobs") + run.rerun_failed_jobs() + except Exception as e: + print(f" rerun_failed_jobs rejected ({e}) - full rerun") + run.rerun() rerun_count += 1 except Exception as e: print(f"Failed to rerun workflow {run.id}: {e}") @@ -455,7 +488,7 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr comment.create_reaction("+1") return True else: - print("No failed or skipped workflows found to rerun.") + print("No failed, skipped, cancelled or timed-out workflows found to rerun.") return False