minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 11 Packages Projects Releases Wiki Activity

[CI] /rerun-failed-ci: rerun cancelled runs and target the newest run per workflow (#34057)

Browse Source
This commit is contained in:
Alison Shao
2026-08-25 20:15:54 -07:00
committed by GitHub
parent 34de1fb47f
commit 3ec22948c1
3 changed files with 43 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/utils/slash_command_handler.py
+41 -8
View File
@@ -351,17 +351,36 @@ def handle_tag_run_ci(
return True
def _latest_run_per_workflow(runs):
"""
Collapse a head_sha's workflow runs to the newest run per workflow.
GitHub can have several runs of the *same* workflow at one commit — a
`synchronize` run superseded by a `labeled` run, or a run cancelled by
`cancel-in-progress` (pr-test.yml sets it) while its replacement is
already in flight. Only the newest one reflects current state; rerunning
the older ones spawns duplicates that fight the live run for runners.
"""
latest = {}
for run in runs:
current = latest.get(run.workflow_id)
if current is None or run.id > current.id:
latest[run.workflow_id] = run
return list(latest.values())
def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=True):
"""
Handles the /rerun-failed-ci command.
Reruns workflows with 'failure' or 'skipped' conclusions.
Reruns workflows that ended in 'failure', 'skipped', 'cancelled' or
'timed_out', restarting only the jobs that didn't succeed.
Returns True if action was taken, False otherwise.
"""
if not user_perms.get("can_rerun_failed_ci", False):
print("Permission denied: can_rerun_failed_ci is false.")
return False
print("Permission granted. Triggering rerun of failed or skipped workflows.")
print("Permission granted. Triggering rerun of unsuccessful workflows.")
# Check if PR has sgl-kernel changes - if so, we may need full reruns
# to ensure sgl-kernel-build-wheels runs and produces fresh artifacts.
@@ -407,7 +426,7 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr
f"Failed to check kernel wheel status: {e} - falling back to full rerun"
)
# Rerun workflows with conclusion=failure or conclusion=skipped.
# Rerun workflows that ended in failure, skipped, cancelled or timed_out.
#
# - failure: use rerun_failed_jobs() which reruns failed jobs *and their
# dependent jobs* (GitHub API). Fast-fail cascades call
@@ -422,17 +441,27 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr
# label-gated workflow, add the missing label (the `labeled` event
# dispatches a fresh run with the current label set); this function
# cannot recover those by rerun alone.
# - cancelled / timed_out: rerun_failed_jobs() as well. GitHub restarts
# every job that didn't succeed — cancelled ones included — plus their
# dependents, and carries the passing jobs over untouched. A full
# rerun here would re-execute dozens of already-green GPU jobs to
# recover one cancelled partition.
# A run cancelled before any job failed has nothing for the endpoint to
# target, so fall back to run.rerun() when it rejects the request.
# - kernel wheel escape: if the PR touches sgl-kernel and not all wheel
# builds are success yet, full-rerun failure runs too — Build Wheel
# lives in pr-test-sgl-kernel.yml, consumers in pr-test.yml, and
# rerun_failed_jobs() is scoped to a single workflow run.
runs = gh_repo.get_workflow_runs(head_sha=head_sha)
runs = _latest_run_per_workflow(gh_repo.get_workflow_runs(head_sha=head_sha))
rerun_count = 0
for run in runs:
if run.status != "completed":
# A newer attempt is still in flight - nothing to recover.
continue
if run.conclusion not in ("failure", "skipped"):
if run.conclusion not in ("failure", "skipped", "cancelled", "timed_out"):
# action_required (fork PR awaiting approval) is deliberately left
# out: it needs an approval, not a rerun.
continue
print(f"Processing {run.conclusion} workflow: {run.name} (ID: {run.id})")
@@ -443,8 +472,12 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr
print(" Full rerun")
run.rerun()
else:
print(" rerun_failed_jobs")
run.rerun_failed_jobs()
try:
print(" rerun_failed_jobs")
run.rerun_failed_jobs()
except Exception as e:
print(f" rerun_failed_jobs rejected ({e}) - full rerun")
run.rerun()
rerun_count += 1
except Exception as e:
print(f"Failed to rerun workflow {run.id}: {e}")
@@ -455,7 +488,7 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr
comment.create_reaction("+1")
return True
else:
print("No failed or skipped workflows found to rerun.")
print("No failed, skipped, cancelled or timed-out workflows found to rerun.")
return False