diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index f84f39c93..d9f3a401b 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -72,6 +72,9 @@ jobs: if: inputs.is_cpu != 'true' runs-on: ${{ inputs.runner_label }} timeout-minutes: 120 + permissions: + contents: read + issues: write env: RUNNER_LABELS: ${{ inputs.runner_label }} SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_label == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }} @@ -81,6 +84,21 @@ jobs: with: ref: ${{ inputs.pr_head_sha || github.sha }} + - name: Mark runner picked up + if: inputs.reply_comment_id != '' && inputs.reply_marker != '' + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then + source /etc/profile.d/sglang-ci.sh + fi + python3 scripts/ci/utils/update_rerun_test_status.py \ + --comment-id "${{ inputs.reply_comment_id }}" \ + --marker "${{ inputs.reply_marker }}" \ + --status running \ + --repo "${{ github.repository }}" + - uses: ./.github/actions/check-maintenance - name: Install dependencies @@ -143,6 +161,9 @@ jobs: if: inputs.is_cpu == 'true' runs-on: ubuntu-latest timeout-minutes: 120 + permissions: + contents: read + issues: write steps: - name: Free disk space run: | @@ -154,6 +175,18 @@ jobs: with: ref: ${{ inputs.pr_head_sha || github.sha }} + - name: Mark runner picked up + if: inputs.reply_comment_id != '' && inputs.reply_marker != '' + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/ci/utils/update_rerun_test_status.py \ + --comment-id "${{ inputs.reply_comment_id }}" \ + --marker "${{ inputs.reply_marker }}" \ + --status running \ + --repo "${{ github.repository }}" + - uses: ./.github/actions/check-maintenance - name: Set up Python @@ -226,9 +259,6 @@ jobs: with: python-version: '3.10' - - name: Install dependencies - run: pip install requests - - name: Write back result to reply comment env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -238,7 +268,7 @@ jobs: else STATUS=failure fi - python3 scripts/ci/utils/write_rerun_test_result.py \ + python3 scripts/ci/utils/update_rerun_test_status.py \ --comment-id "${{ inputs.reply_comment_id }}" \ --marker "${{ inputs.reply_marker }}" \ --status "$STATUS" \ diff --git a/scripts/ci/utils/update_rerun_test_status.py b/scripts/ci/utils/update_rerun_test_status.py new file mode 100755 index 000000000..448c45417 --- /dev/null +++ b/scripts/ci/utils/update_rerun_test_status.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +""" +Update the per-batch status icon in a /rerun-test reply comment. + +State machine for one batch line (anchored by a unique HTML-comment marker +written by the slash-command handler): + + dispatched ⏳ ... (handler, on dispatch) + running 🔄 ... (start-beacon, on the test runner) + done ✅/❌ ... (finalizer, after the test job) + +The leading 🚀 on the line is the visual anchor that this came from a +slash-command trigger and is preserved across all states. + +Idempotency: +- :done marker present -> no-op (covers reruns and start-after-finalizer race) +- running and line already has 🔄 -> no-op +- marker not found after retries -> warn and exit 0 so a single placeholder + glitch does not amplify into N noisy job failures. + +Concurrent updates against the same comment from different batches can race +on the body (read-modify-write of the same field). The finalizer +serializes itself via job-level concurrency; the beacon does not, because +splitting it into its own job would re-queue the GPU runner. Worst case +for the beacon race is one missed 🔄 flicker - comment stays consistent. +""" + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.request + +RETRY_DELAYS_SEC = [0, 5, 15] + +STATUS_ICONS = { + "running": "🔄", + "success": "✅", + "failure": "❌", +} + +TERMINAL_STATUSES = {"success", "failure"} + + +def gh_request(method, url, token, body=None): + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(url, data=data, method=method, headers=headers) + if data is not None: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=15) as resp: + return resp.status, resp.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--comment-id", required=True, type=int) + ap.add_argument( + "--marker", required=True, help="Per-batch marker, e.g. " + ) + ap.add_argument( + "--status", + required=True, + choices=list(STATUS_ICONS.keys()), + ) + ap.add_argument("--repo", required=True, help="owner/repo") + args = ap.parse_args() + + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("ERROR: GITHUB_TOKEN not set") + return 1 + + icon = STATUS_ICONS[args.status] + is_terminal = args.status in TERMINAL_STATUSES + done_marker = args.marker.replace("-->", ":done-->") + + url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}" + + body = None + for attempt, delay in enumerate(RETRY_DELAYS_SEC): + if delay: + time.sleep(delay) + status, text = gh_request("GET", url, token) + if status != 200: + print(f"GET failed: {status} {text}") + return 1 + body = json.loads(text).get("body") or "" + if done_marker in body: + print(f"Marker {done_marker} already present; nothing to do.") + return 0 + if args.marker in body: + break + print( + f"Marker {args.marker} not found " + f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry." + ) + else: + print( + f"WARNING: marker {args.marker} not found after " + f"{len(RETRY_DELAYS_SEC)} attempts; skipping. " + f"The handler may have failed to edit the placeholder comment." + ) + return 0 + + new_lines = [] + for line in body.splitlines(keepends=True): + if args.marker in line: + if not is_terminal and icon in line: + print(f"Line already has {icon}; nothing to do.") + return 0 + for prior in ("⏳", "🔄"): + if prior in line: + line = line.replace(prior, icon, 1) + break + if is_terminal: + line = line.replace(args.marker, done_marker) + new_lines.append(line) + + new_body = "".join(new_lines) + status, text = gh_request("PATCH", url, token, body={"body": new_body}) + if status != 200: + print(f"PATCH failed: {status} {text}") + return 1 + print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ci/utils/write_rerun_test_result.py b/scripts/ci/utils/write_rerun_test_result.py deleted file mode 100755 index a3f06b9b7..000000000 --- a/scripts/ci/utils/write_rerun_test_result.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -""" -Write back the result of a Rerun Test workflow run into its parent reply -comment, by replacing the hourglass placeholder on the marker's line with -a success/failure icon. The leading rocket on the line is preserved as a -visual anchor that this line came from a slash-command trigger. - -Concurrent dispatches against the same reply comment are serialized at the -GitHub Actions layer via job-level `concurrency`, so a simple -read-modify-write here is sufficient. Marker is rewritten to a `:done` -variant to make the operation idempotent against accidental reruns. -""" - -import argparse -import os -import sys -import time - -import requests - -# When the marker isn't found, retry a few times before giving up. The -# usual cause is a brief race where the dispatched workflow finishes its -# writeback step before the handler edits the placeholder comment with the -# final body containing markers. If retries don't help (e.g. the handler -# failed to edit the placeholder at all), we still return 0 — failing here -# would amplify a single placeholder-edit failure into N noisy finalizer -# job failures, one per dispatched batch. -RETRY_DELAYS_SEC = [0, 5, 15] - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument("--comment-id", required=True, type=int) - ap.add_argument( - "--marker", required=True, help="Per-batch marker, e.g. " - ) - ap.add_argument("--status", required=True, choices=["success", "failure"]) - ap.add_argument("--repo", required=True, help="owner/repo") - args = ap.parse_args() - - token = os.environ.get("GITHUB_TOKEN") - if not token: - print("ERROR: GITHUB_TOKEN not set") - return 1 - - icon = "✅" if args.status == "success" else "❌" - done_marker = args.marker.replace("-->", ":done-->") - - headers = { - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - } - url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}" - - body = None - for attempt, delay in enumerate(RETRY_DELAYS_SEC): - if delay: - time.sleep(delay) - resp = requests.get(url, headers=headers, timeout=15) - if resp.status_code != 200: - print(f"GET failed: {resp.status_code} {resp.text}") - return 1 - body = resp.json().get("body") or "" - if done_marker in body: - print(f"Marker {done_marker} already present; nothing to do.") - return 0 - if args.marker in body: - break - print( - f"Marker {args.marker} not found " - f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry." - ) - else: - print( - f"WARNING: marker {args.marker} not found after " - f"{len(RETRY_DELAYS_SEC)} attempts; skipping writeback. " - f"The handler may have failed to edit the placeholder comment." - ) - return 0 - - new_lines = [] - for line in body.splitlines(keepends=True): - if args.marker in line: - line = line.replace("⏳", icon, 1) - line = line.replace(args.marker, done_marker) - new_lines.append(line) - new_body = "".join(new_lines) - - patch = requests.patch( - url, - headers=headers, - json={"body": new_body}, - timeout=15, - ) - if patch.status_code != 200: - print(f"PATCH failed: {patch.status_code} {patch.text}") - return 1 - print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}") - return 0 - - -if __name__ == "__main__": - sys.exit(main())