diff --git a/.claude/skills/ci-workflow-guide/SKILL.md b/.claude/skills/ci-workflow-guide/SKILL.md index e660e9333..0a6ed7b30 100644 --- a/.claude/skills/ci-workflow-guide/SKILL.md +++ b/.claude/skills/ci-workflow-guide/SKILL.md @@ -388,6 +388,8 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage} | `/rerun-failed-ci` | Reruns failed jobs in the latest workflow run | | `/tag-and-rerun-ci` | Adds `run-ci` label + reruns failed | | `/tag-and-rerun-ci extra` | Adds both `run-ci` and `run-ci-extra` labels + reruns failed | +| `/run-full-ci` | Short form of `/tag-and-rerun-ci extra` (baseline + extra). Alias: `/rerun-full-ci` | +| `/run-extra-ci` | Adds both labels + reruns **only** `PR Test Extra`, leaving baseline runs alone. Alias: `/rerun-extra-ci` | | `/rerun-test [ ...]` | Reruns specific test file(s) via `rerun-test.yml`. A file arg containing a glob metacharacter (`*`, `?`, `[...]`) expands against `test/registered/` and the multimodal test dir to every matching `test_*.py` (e.g. `/rerun-test test_*backend*.py` — wrap in backticks so GitHub doesn't italicize the `*`); matches are deduped, grouped by dispatch shape, and can't carry a `::test` selector. No match → single ⛔ reply, nothing dispatched. Each reply echoes its originating command (`Results for …`) so concurrent commands stay distinguishable | | `/rerun-group [ ...]` | Expands registered test groups, then reuses `/rerun-test` | diff --git a/.github/FOLDER_README.md b/.github/FOLDER_README.md index f987d4d92..f951adfa8 100644 --- a/.github/FOLDER_README.md +++ b/.github/FOLDER_README.md @@ -12,8 +12,8 @@ Recognized permission keys: | Key | Grants | | --- | --- | -| `can_tag_run_ci_label` | `/tag-run-ci-label`, `/tag-and-rerun-ci` | -| `can_rerun_failed_ci` | `/rerun-failed-ci`, `/tag-and-rerun-ci` | +| `can_tag_run_ci_label` | `/tag-run-ci-label`, `/tag-and-rerun-ci`, `/run-full-ci`, `/run-extra-ci` | +| `can_rerun_failed_ci` | `/rerun-failed-ci`, `/tag-and-rerun-ci`, `/run-full-ci`, `/run-extra-ci` | | `cooldown_interval_minutes` | rate limit in `pr-gate.yml`; `0` also grants `/rerun-test`, `/rerun-group` | `/rerun-test` and `/rerun-group` are gated on the commenter alone: either diff --git a/.github/workflows/slash-command-handler.yml b/.github/workflows/slash-command-handler.yml index 3c48d1aa7..d8239f8fb 100644 --- a/.github/workflows/slash-command-handler.yml +++ b/.github/workflows/slash-command-handler.yml @@ -19,6 +19,10 @@ jobs: (contains(github.event.comment.body, '/tag-run-ci-label') || contains(github.event.comment.body, '/rerun-failed-ci') || contains(github.event.comment.body, '/tag-and-rerun-ci') || + contains(github.event.comment.body, '/run-full-ci') || + contains(github.event.comment.body, '/run-extra-ci') || + contains(github.event.comment.body, '/rerun-full-ci') || + contains(github.event.comment.body, '/rerun-extra-ci') || contains(github.event.comment.body, '/rerun-group') || contains(github.event.comment.body, '/rerun-test')) runs-on: ubuntu-latest diff --git a/docs/docs/developer_guide/contribution_guide.mdx b/docs/docs/developer_guide/contribution_guide.mdx index 3a74e13cf..527a10945 100644 --- a/docs/docs/developer_guide/contribution_guide.mdx +++ b/docs/docs/developer_guide/contribution_guide.mdx @@ -122,14 +122,17 @@ For CI to run on a pull request, it must have the "run-ci" label. Authorized use - `/tag-run-ci-label`: Adds the "run-ci" label. Only **future** commits trigger CI; the current commit is unaffected. Add the `extra` argument (`/tag-run-ci-label extra`) to additionally apply the "run-ci-extra" label, opting the PR into the extra test workflow (`pr-test-extra.yml`). - `/rerun-failed-ci`: Reruns workflows from the latest commit with conclusion **failed, skipped, cancelled, or timed out**. - `/tag-and-rerun-ci`: Runs both. Use this on a fresh PR to kick off CI on the current commit — `/tag-run-ci-label` alone won't. Accepts the same `extra` argument (`/tag-and-rerun-ci extra`). +- `/run-full-ci`: Baseline **and** extra CI. Short form of `/tag-and-rerun-ci extra` — applies both labels, then reruns everything at the current commit that didn't succeed. +- `/run-extra-ci`: Extra CI **only**. Applies both labels, then reruns just `PR Test Extra`, leaving baseline CI runs alone. Use it when baseline is already green and you only need the extra tier. - `/rerun-test [ ...]`: Reruns one or more specific tests directly. A spec may select a file, class, or method using `::[.]`. Multiple specs and file globs are supported. `--changed` (short form `-c`) adds every test file the PR itself adds or modifies (under `test/registered/` or the multimodal test directory), so a PR that touches several tests can rerun them all without listing them. Examples: `/rerun-test test_srt_endpoint.py`, `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`, `/rerun-test test_a.py test_b.py`, `/rerun-test test_*backend*.py`, `/rerun-test --changed`, or `/rerun-test -c`. -- `/rerun-group [ ...]`: Expands one or more registered test groups (for example, `/rerun-group hicache`) and dispatches their tests through the same selective-rerun workflow. +- `/rerun-group [ ...]`: Expands one or more registered test groups (for example, `/rerun-group rust-server`) and dispatches their tests through the same selective-rerun workflow. Groups are defined in [`scripts/ci/rerun_test_groups.json`](https://github.com/sgl-project/sglang/blob/main/scripts/ci/rerun_test_groups.json). The rerun commands have the following permission rules: | Command | PR author | Other users | Additional rule for fork PRs | | --- | --- | --- | --- | | `/rerun-failed-ci` | Always allowed on their own PR | Must have `can_rerun_failed_ci` in `CI_PERMISSIONS.json` | None | +| `/run-full-ci`, `/run-extra-ci` | Not auto-allowed -- these apply labels, which the PR-author allowance does not cover | Must have both `can_tag_run_ci_label` and `can_rerun_failed_ci` in `CI_PERMISSIONS.json` (every listed user has both) | None | | `/rerun-test`, `/rerun-group` | No special allowance; judged as any other commenter | Must have `cooldown_interval_minutes: 0` in `CI_PERMISSIONS.json`, or `write`/`admin` repository permission | None -- the same rule applies wherever the PR comes from | diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index b2f3f1ab8..dad457678 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -32,6 +32,8 @@ PRECISION_BASELINE_TEST = "registered/debug_utils/test_nightly_precision_regress PRECISION_BASELINE_REFRESH_FLAG = "--refresh-precision-baseline" CHANGED_TESTS_FLAG = "--changed" CHANGED_TESTS_SHORT_FLAG = "-c" +# Workflow `name:` field, which is what the runs API reports (not the filename). +EXTRA_WORKFLOW_NAME = "PR Test Extra" MAINTENANCE_ISSUE_NUMBER = 21065 @@ -495,6 +497,94 @@ def handle_rerun_failed_ci(gh_repo, pr, comment, user_perms, react_on_success=Tr return False +def _run_extra_ci(gh_repo, pr): + """ + Make `PR Test Extra` run for the current head SHA. + + Assumes `run-ci` + `run-ci-extra` are already on the PR. pr-test-extra.yml + gates on them through call-gate -> pr-gate.yml, which live-fetches the + label set at run time, so re-running a run that was red at the gate picks + up labels added seconds earlier. + + Deliberately rerun-only, with no workflow_dispatch fallback: the extra + workflow's dispatch inputs carry no `pr_head_sha`, so a fork PR could only + be dispatched against `main` — testing the wrong tree. A pull_request run + exists at essentially every head SHA anyway, so the missing-run case is + reported rather than papered over. + + Returns (acted, message). + """ + extra_runs = [ + run + for run in gh_repo.get_workflow_runs(head_sha=pr.head.sha) + if run.name == EXTRA_WORKFLOW_NAME + ] + if not extra_runs: + return ( + False, + f"No `{EXTRA_WORKFLOW_NAME}` run exists for this commit, so there is " + "nothing to re-run. Push a commit to create one.", + ) + + # Same newest-wins rule handle_rerun_failed_ci uses: older runs at this SHA + # are superseded and re-running them fights the live run for runners. + run = _latest_run_per_workflow(extra_runs)[0] + + if run.status != "completed": + return ( + True, + f"⏳ [`{EXTRA_WORKFLOW_NAME}`]({run.html_url}) is already " + f"{run.status} for this commit; left it alone.", + ) + + try: + if run.conclusion == "success": + # Nothing failed, so there are no jobs for rerun_failed_jobs() to + # target — the ask is an explicit re-run of green work. + print(f" Full rerun of successful run {run.id}") + run.rerun() + else: + try: + print(f" rerun_failed_jobs on {run.id} ({run.conclusion})") + run.rerun_failed_jobs() + except Exception as e: + print(f" rerun_failed_jobs rejected ({e}) - full rerun") + run.rerun() + except Exception as e: + return False, f"Failed to re-run [`{EXTRA_WORKFLOW_NAME}`]({run.html_url}): {e}" + + return True, f"🚀 Re-running [`{EXTRA_WORKFLOW_NAME}`]({run.html_url})." + + +def handle_run_extra_ci(gh_repo, pr, comment, user_perms): + """ + Handles /run-extra-ci: label the PR for extra CI and re-run only + `PR Test Extra`. Baseline CI runs are left untouched. + + Gated on the same two permissions as the steps it performs — the label add + (can_tag_run_ci_label) and the re-run (can_rerun_failed_ci) — rather than a + new key, so every user who can already drive CI can drive extra CI. + Returns True if action was taken, False otherwise. + """ + if not user_perms.get("can_rerun_failed_ci", False): + print("Permission denied: can_rerun_failed_ci is false.") + return False + + tagged = handle_tag_run_ci( + gh_repo, pr, comment, user_perms, react_on_success=False, tag_extra=True + ) + if not tagged: + return False + + print("Waiting 5 seconds for labels to propagate...") + time.sleep(5) + + acted, message = _run_extra_ci(gh_repo, pr) + pr.create_issue_comment(message if acted else f"⛔ {message}") + comment.create_reaction("+1" if acted else "confused") + return acted + + MULTIMODAL_TEST_DIR = "python/sglang/multimodal_gen/test" MULTIMODAL_PATH_TO_RUNNER = { @@ -1575,15 +1665,28 @@ def main(): tokens = first_line.split() tag_extra = len(tokens) > 1 and "extra" in tokens[1:] + # /run-full-ci is the short, spelled-out form of `/tag-and-rerun-ci extra` + # and shares its branch below. /rerun-* spellings are accepted silently as + # aliases: they sit one keystroke from the documented name and from the + # neighbouring /rerun-* commands, and failing them closed would just look + # like the bot ignoring the comment. + is_full_ci = first_line.startswith(("/run-full-ci", "/rerun-full-ci")) + is_extra_ci = first_line.startswith(("/run-extra-ci", "/rerun-extra-ci")) + if is_full_ci: + tag_extra = True + if first_line.startswith("/tag-run-ci-label"): handle_tag_run_ci(repo, pr, comment, user_perms, tag_extra=tag_extra) + elif is_extra_ci: + handle_run_extra_ci(repo, pr, comment, user_perms) + elif first_line.startswith("/rerun-failed-ci"): handle_rerun_failed_ci(repo, pr, comment, user_perms) - elif first_line.startswith("/tag-and-rerun-ci"): + elif first_line.startswith("/tag-and-rerun-ci") or is_full_ci: # Perform both actions, but suppress individual reactions - print(f"Processing combined command: /tag-and-rerun-ci (tag_extra={tag_extra})") + print(f"Processing combined command: {first_line} (tag_extra={tag_extra})") tagged = handle_tag_run_ci( repo, pr, comment, user_perms, react_on_success=False, tag_extra=tag_extra