From e0273dcd316be010bbf461879f2ee5c4c7ab50f3 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Tue, 19 May 2026 05:15:55 -0700 Subject: [PATCH] pr-test-extra: re-trigger on labeled event (#25732) --- .claude/skills/ci-workflow-guide/SKILL.md | 18 ++++++- .github/workflows/pr-test-extra.yml | 12 +++++ .../developer_guide/contribution_guide.mdx | 4 +- .../ascend-npus/ascend_contribution_guide.mdx | 4 +- scripts/ci/utils/slash_command_handler.py | 52 ++++++++++++++++--- 5 files changed, 76 insertions(+), 14 deletions(-) diff --git a/.claude/skills/ci-workflow-guide/SKILL.md b/.claude/skills/ci-workflow-guide/SKILL.md index 3b71881ca..00a7991c9 100644 --- a/.claude/skills/ci-workflow-guide/SKILL.md +++ b/.claude/skills/ci-workflow-guide/SKILL.md @@ -22,6 +22,7 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched, | File | Role | |------|------| | `.github/workflows/pr-test.yml` | Main workflow — all stages, jobs, conditions, matrix definitions | +| `.github/workflows/pr-test-extra.yml` | Extra workflow — gated by BOTH `run-ci` and `run-ci-extra` labels | | `.github/workflows/pr-gate.yml` | PR gating: draft check, `run-ci` label, per-user rate limiting | | `.github/actions/check-pr-test-health/action.yml` | Cross-job fast-fail: queries API for any failed job | | `.github/actions/wait-for-jobs/action.yml` | Stage gating: polls API until stage jobs complete | @@ -381,10 +382,23 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage} | Command | Effect | |---------|--------| | `/tag-run-ci-label` | Adds `run-ci` label to PR | +| `/tag-run-ci-label extra` | Adds both `run-ci` and `run-ci-extra` labels | | `/rerun-failed-ci` | Reruns failed jobs in the latest workflow run | -| `/tag-and-rerun-ci` | Adds label + reruns | -| `/rerun-stage ` | Dispatches `pr-test.yml` with `target_stage=` | +| `/tag-and-rerun-ci` | Adds `run-ci` label + reruns failed | +| `/tag-and-rerun-ci extra` | Adds both `run-ci` and `run-ci-extra` labels + reruns failed | +| `/rerun-stage ` | Deprecated; posts deprecation notice | | `/rerun-test ` | Reruns a specific test file via `rerun-test.yml` | | `/rerun-group [ ...]` | Expands registered test groups, then reuses `/rerun-test` | Handled by `scripts/ci/utils/slash_command_handler.py` → `.github/workflows/slash-command-handler.yml`. + +### Label-gated workflow dispatch (pr-test, pr-test-extra) + +`pr-test.yml` and `pr-test-extra.yml` both listen for `pull_request.labeled` (in addition to `opened`/`synchronize`/`reopened`). The `check-changes.if` gate has two clauses: + +1. **For `labeled` events**: the just-added label must be one of the gating labels (`run-ci` for pr-test, `run-ci` or `run-ci-extra` for pr-test-extra) — otherwise every unrelated label addition would dispatch a full CI run. +2. **All events**: the PR must currently carry the required labels. + +This is what lets `/tag-run-ci-label` (and the `extra` variant) trigger a fresh CI run without an extra push. + +**Caveat — skipped runs cannot be un-skipped by `run.rerun()`:** GitHub's rerun API reuses the original event payload, so rerunning a `pull_request`-event run that was skipped because of missing labels will skip again (label set in the frozen payload doesn't update). The only way to recover a label-skipped run is to add the missing label, which fires a fresh `labeled` event with the current label set. `handle_rerun_failed_ci` in the slash handler is for rerunning failed/non-label-skipped runs; it cannot revive label-skipped ones. diff --git a/.github/workflows/pr-test-extra.yml b/.github/workflows/pr-test-extra.yml index 8b8aa4bd7..d14c95d8f 100644 --- a/.github/workflows/pr-test-extra.yml +++ b/.github/workflows/pr-test-extra.yml @@ -12,6 +12,11 @@ name: PR Test Extra on: pull_request: branches: [main] + # `labeled` lets the workflow re-fire when `run-ci-extra` (or `run-ci`) + # is added after the latest push — otherwise GHA leaves the stale + # skipped run in place and there's no way to enable extra without + # another push. See check-changes.if below for the matching guard. + types: [opened, synchronize, reopened, labeled] workflow_dispatch: inputs: force_continue_on_error: @@ -70,10 +75,17 @@ jobs: # (workflow_dispatch / workflow_call) always run. When this job is # skipped by the gate, every downstream caller stub naturally skips # because its needs do not resolve. + # + # For `labeled` events we additionally require the just-added label to be + # one of the two gating labels — otherwise every unrelated label addition + # would dispatch a full CI run. check-changes: if: | github.event_name != 'pull_request' || ( + (github.event.action != 'labeled' || + github.event.label.name == 'run-ci' || + github.event.label.name == 'run-ci-extra') && contains(github.event.pull_request.labels.*.name, 'run-ci') && contains(github.event.pull_request.labels.*.name, 'run-ci-extra') ) diff --git a/docs_new/docs/developer_guide/contribution_guide.mdx b/docs_new/docs/developer_guide/contribution_guide.mdx index 741550a41..1bfe9495c 100644 --- a/docs_new/docs/developer_guide/contribution_guide.mdx +++ b/docs_new/docs/developer_guide/contribution_guide.mdx @@ -114,9 +114,9 @@ Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: -- `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. +- `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. Add the `extra` argument (`/tag-run-ci-label extra`) to additionally apply the "run-ci-extra" label, opting the PR into the extra test workflow (`pr-test-extra.yml`). - `/rerun-failed-ci`: Reruns the failed or flaky tests from the most recent commit. -- `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. +- `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. Accepts the same `extra` argument (`/tag-and-rerun-ci extra`). - `/rerun-stage `: Reruns a specific test stage without waiting for its dependencies. This is useful when you want to quickly validate a fix for a specific test failure instead of waiting ~30 minutes for preceding stages to complete. - `/rerun-test [ ...]`: Reruns one or more specific tests directly, bypassing stage boundaries. Each `` is pytest-style `::[.]` (the `::TestClass` and `.` parts are optional). The handler resolves each spec, groups specs by their registered runner-label, and dispatches one [Rerun Test workflow](https://github.com/sgl-project/sglang/actions/workflows/rerun-test.yml) per group. Examples: `/rerun-test test_srt_endpoint.py`, `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`, `/rerun-test test_a.py test_b.py` (multiple at once). diff --git a/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx b/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx index 3005aa246..e85f8694b 100644 --- a/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx +++ b/docs_new/docs/hardware-platforms/ascend-npus/ascend_contribution_guide.mdx @@ -94,9 +94,9 @@ Users with permission are listed in the [CI_PERMISSIONS.json](https://github.com For CI to run on a pull request, it must have the "run-ci" label. Authorized users can add the label or rerun failed tests by commenting on the PR with one of these commands: -- `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. +- `/tag-run-ci-label`: Adds the "run-ci" label. Every future commit will trigger CI. Add the `extra` argument (`/tag-run-ci-label extra`) to additionally apply the "run-ci-extra" label, opting the PR into the extra test workflow (`pr-test-extra.yml`). - `/rerun-failed-ci`: Reruns the failed or flaky tests from the most recent commit. -- `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. +- `/tag-and-rerun-ci`: A single command that performs both `/tag-run-ci-label` and `/rerun-failed-ci`. Accepts the same `extra` argument (`/tag-and-rerun-ci extra`). - `/rerun-stage `: Reruns a specific test stage without waiting for its dependencies. This is useful when you want to quickly validate a fix for a specific test failure instead of waiting ~30 minutes for preceding stages to complete. If you have permission, the [Slash Command Handler](https://github.com/sgl-project/sglang/actions/workflows/slash-command-handler.yml) will run your command and react with a 👍 to your comment. It may take up to a few minutes for the reaction to appear. Here’s a usage [example](https://github.com/sgl-project/sglang/pull/14253#issuecomment-3599509302). diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index ee67a8a6c..44c0dacfe 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -282,23 +282,46 @@ def has_sgl_kernel_changes(pr): return False -def handle_tag_run_ci(gh_repo, pr, comment, user_perms, react_on_success=True): +def handle_tag_run_ci( + gh_repo, pr, comment, user_perms, react_on_success=True, tag_extra=False +): """ Handles the /tag-run-ci-label command. + + When tag_extra is True (triggered by the `extra` argument), also adds the + `run-ci-extra` label. pr-test-extra.yml gates on BOTH `run-ci` and + `run-ci-extra`, so both must be present for the extra workflow to run — + we always add `run-ci` alongside `run-ci-extra`. Reuses the same + `can_tag_run_ci_label` permission. + + How fresh runs get dispatched: pr-test.yml and pr-test-extra.yml both + include `labeled` in `on.pull_request.types`, so adding a label fires a + new `pull_request.labeled` event with the up-to-date label set in its + payload, which spawns a fresh workflow run that satisfies the + `check-changes.if` gate. Note that this is the ONLY way to "un-skip" a + label-gated run — `run.rerun()` on a previously-skipped pull_request run + reuses the original event payload (frozen labels), so it would skip + again. handle_rerun_failed_ci can't recover label-skipped runs; the + labeled event is the recovery mechanism. + Returns True if action was taken, False otherwise. """ if not user_perms.get("can_tag_run_ci_label", False): print("Permission denied: can_tag_run_ci_label is false.") return False - print("Permission granted. Adding 'run-ci' label.") - pr.add_to_labels("run-ci") + labels = ["run-ci"] + if tag_extra: + labels.append("run-ci-extra") + print(f"Permission granted. Adding labels: {labels}.") + for label in labels: + pr.add_to_labels(label) if react_on_success: comment.create_reaction("+1") - print("Label added and comment reacted.") + print("Labels added and comment reacted.") else: - print("Label added (reaction suppressed).") + print("Labels added (reaction suppressed).") return True @@ -366,6 +389,14 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr # core.setFailed(...) so their conclusion is "failure" and are covered. # - skipped: the entire run was skipped (no jobs ran), so there are no # failed jobs for rerun_failed_jobs() to target. Use run.rerun(). + # + # Caveat: GitHub's `run.rerun()` reuses the original event payload, so + # reruns of `pull_request`-event runs that were skipped because their + # `if` evaluated to false (e.g. missing label) will skip again — the + # label set in the frozen payload doesn't update. To un-skip a + # label-gated workflow, add the missing label (the `labeled` event + # dispatches a fresh run with the current label set); this function + # cannot recover those by rerun alone. # - kernel wheel escape: if the PR touches sgl-kernel and not all wheel # builds are success yet, full-rerun failure runs too — Build Wheel # lives in pr-test-sgl-kernel.yml, consumers in pr-test.yml, and @@ -1071,19 +1102,24 @@ def main(): # 4. Parse Command and Execute first_line = comment_body.split("\n")[0].strip() + # `extra` argument opts in to also tagging `run-ci-extra`. Both + # `/tag-run-ci-label extra` and `/tag-and-rerun-ci extra` share this + # parser so the surface is symmetric. + tokens = first_line.split() + tag_extra = len(tokens) > 1 and "extra" in tokens[1:] if first_line.startswith("/tag-run-ci-label"): - handle_tag_run_ci(repo, pr, comment, user_perms) + handle_tag_run_ci(repo, pr, comment, user_perms, tag_extra=tag_extra) elif first_line.startswith("/rerun-failed-ci"): handle_rerun_failed_ci(repo, pr, comment, user_perms) elif first_line.startswith("/tag-and-rerun-ci"): # Perform both actions, but suppress individual reactions - print("Processing combined command: /tag-and-rerun-ci") + print(f"Processing combined command: /tag-and-rerun-ci (tag_extra={tag_extra})") tagged = handle_tag_run_ci( - repo, pr, comment, user_perms, react_on_success=False + repo, pr, comment, user_perms, react_on_success=False, tag_extra=tag_extra ) # Wait for the label to propagate before triggering rerun