rerun-test: show running state in reply comment (#25018)

.github/workflows/rerun-test.yml

@@ -72,6 +72,9 @@ jobs:
     if: inputs.is_cpu != 'true'
     runs-on: ${{ inputs.runner_label }}
     timeout-minutes: 120
+    permissions:
+      contents: read
+      issues: write
     env:
       RUNNER_LABELS: ${{ inputs.runner_label }}
       SGLANG_CI_RDMA_ALL_DEVICES: ${{ inputs.runner_label == '8-gpu-h20' && 'mlx5_1,mlx5_2,mlx5_3,mlx5_4' || '' }}
@@ -81,6 +84,21 @@ jobs:
         with:
           ref: ${{ inputs.pr_head_sha || github.sha }}
 
+      - name: Mark runner picked up
+        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ "${{ inputs.runner_label }}" == "1-gpu-5090" ]]; then
+            source /etc/profile.d/sglang-ci.sh
+          fi
+          python3 scripts/ci/utils/update_rerun_test_status.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status running \
+            --repo "${{ github.repository }}"
+
       - uses: ./.github/actions/check-maintenance
 
       - name: Install dependencies
@@ -143,6 +161,9 @@ jobs:
     if: inputs.is_cpu == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 120
+    permissions:
+      contents: read
+      issues: write
     steps:
       - name: Free disk space
         run: |
@@ -154,6 +175,18 @@ jobs:
         with:
           ref: ${{ inputs.pr_head_sha || github.sha }}
 
+      - name: Mark runner picked up
+        if: inputs.reply_comment_id != '' && inputs.reply_marker != ''
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/ci/utils/update_rerun_test_status.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status running \
+            --repo "${{ github.repository }}"
+
       - uses: ./.github/actions/check-maintenance
 
       - name: Set up Python
@@ -226,9 +259,6 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install dependencies
-        run: pip install requests
-
       - name: Write back result to reply comment
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -238,7 +268,7 @@ jobs:
           else
             STATUS=failure
           fi
-          python3 scripts/ci/utils/write_rerun_test_result.py \
+          python3 scripts/ci/utils/update_rerun_test_status.py \
             --comment-id "${{ inputs.reply_comment_id }}" \
             --marker "${{ inputs.reply_marker }}" \
             --status "$STATUS" \

scripts/ci/utils/update_rerun_test_status.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""
+Update the per-batch status icon in a /rerun-test reply comment.
+
+State machine for one batch line (anchored by a unique HTML-comment marker
+written by the slash-command handler):
+
+  dispatched   ⏳ ... <!--rrt:i-->          (handler, on dispatch)
+  running      🔄 ... <!--rrt:i-->          (start-beacon, on the test runner)
+  done         ✅/❌ ... <!--rrt:i:done-->  (finalizer, after the test job)
+
+The leading 🚀 on the line is the visual anchor that this came from a
+slash-command trigger and is preserved across all states.
+
+Idempotency:
+- :done marker present  -> no-op (covers reruns and start-after-finalizer race)
+- running and line already has 🔄  -> no-op
+- marker not found after retries  -> warn and exit 0 so a single placeholder
+  glitch does not amplify into N noisy job failures.
+
+Concurrent updates against the same comment from different batches can race
+on the body (read-modify-write of the same field). The finalizer
+serializes itself via job-level concurrency; the beacon does not, because
+splitting it into its own job would re-queue the GPU runner. Worst case
+for the beacon race is one missed 🔄 flicker - comment stays consistent.
+"""
+
+import argparse
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.request
+
+RETRY_DELAYS_SEC = [0, 5, 15]
+
+STATUS_ICONS = {
+    "running": "🔄",
+    "success": "✅",
+    "failure": "❌",
+}
+
+TERMINAL_STATUSES = {"success", "failure"}
+
+
+def gh_request(method, url, token, body=None):
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, method=method, headers=headers)
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    try:
+        with urllib.request.urlopen(req, timeout=15) as resp:
+            return resp.status, resp.read().decode()
+    except urllib.error.HTTPError as e:
+        return e.code, e.read().decode()
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--comment-id", required=True, type=int)
+    ap.add_argument(
+        "--marker", required=True, help="Per-batch marker, e.g. <!--rrt:0-->"
+    )
+    ap.add_argument(
+        "--status",
+        required=True,
+        choices=list(STATUS_ICONS.keys()),
+    )
+    ap.add_argument("--repo", required=True, help="owner/repo")
+    args = ap.parse_args()
+
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        print("ERROR: GITHUB_TOKEN not set")
+        return 1
+
+    icon = STATUS_ICONS[args.status]
+    is_terminal = args.status in TERMINAL_STATUSES
+    done_marker = args.marker.replace("-->", ":done-->")
+
+    url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}"
+
+    body = None
+    for attempt, delay in enumerate(RETRY_DELAYS_SEC):
+        if delay:
+            time.sleep(delay)
+        status, text = gh_request("GET", url, token)
+        if status != 200:
+            print(f"GET failed: {status} {text}")
+            return 1
+        body = json.loads(text).get("body") or ""
+        if done_marker in body:
+            print(f"Marker {done_marker} already present; nothing to do.")
+            return 0
+        if args.marker in body:
+            break
+        print(
+            f"Marker {args.marker} not found "
+            f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry."
+        )
+    else:
+        print(
+            f"WARNING: marker {args.marker} not found after "
+            f"{len(RETRY_DELAYS_SEC)} attempts; skipping. "
+            f"The handler may have failed to edit the placeholder comment."
+        )
+        return 0
+
+    new_lines = []
+    for line in body.splitlines(keepends=True):
+        if args.marker in line:
+            if not is_terminal and icon in line:
+                print(f"Line already has {icon}; nothing to do.")
+                return 0
+            for prior in ("⏳", "🔄"):
+                if prior in line:
+                    line = line.replace(prior, icon, 1)
+                    break
+            if is_terminal:
+                line = line.replace(args.marker, done_marker)
+        new_lines.append(line)
+
+    new_body = "".join(new_lines)
+    status, text = gh_request("PATCH", url, token, body={"body": new_body})
+    if status != 200:
+        print(f"PATCH failed: {status} {text}")
+        return 1
+    print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/utils/write_rerun_test_result.py

@@ -1,104 +0,0 @@
-#!/usr/bin/env python3
-"""
-Write back the result of a Rerun Test workflow run into its parent reply
-comment, by replacing the hourglass placeholder on the marker's line with
-a success/failure icon. The leading rocket on the line is preserved as a
-visual anchor that this line came from a slash-command trigger.
-
-Concurrent dispatches against the same reply comment are serialized at the
-GitHub Actions layer via job-level `concurrency`, so a simple
-read-modify-write here is sufficient. Marker is rewritten to a `:done`
-variant to make the operation idempotent against accidental reruns.
-"""
-
-import argparse
-import os
-import sys
-import time
-
-import requests
-
-# When the marker isn't found, retry a few times before giving up. The
-# usual cause is a brief race where the dispatched workflow finishes its
-# writeback step before the handler edits the placeholder comment with the
-# final body containing markers. If retries don't help (e.g. the handler
-# failed to edit the placeholder at all), we still return 0 — failing here
-# would amplify a single placeholder-edit failure into N noisy finalizer
-# job failures, one per dispatched batch.
-RETRY_DELAYS_SEC = [0, 5, 15]
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--comment-id", required=True, type=int)
-    ap.add_argument(
-        "--marker", required=True, help="Per-batch marker, e.g. <!--rrt:0-->"
-    )
-    ap.add_argument("--status", required=True, choices=["success", "failure"])
-    ap.add_argument("--repo", required=True, help="owner/repo")
-    args = ap.parse_args()
-
-    token = os.environ.get("GITHUB_TOKEN")
-    if not token:
-        print("ERROR: GITHUB_TOKEN not set")
-        return 1
-
-    icon = "✅" if args.status == "success" else "❌"
-    done_marker = args.marker.replace("-->", ":done-->")
-
-    headers = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github+json",
-        "X-GitHub-Api-Version": "2022-11-28",
-    }
-    url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}"
-
-    body = None
-    for attempt, delay in enumerate(RETRY_DELAYS_SEC):
-        if delay:
-            time.sleep(delay)
-        resp = requests.get(url, headers=headers, timeout=15)
-        if resp.status_code != 200:
-            print(f"GET failed: {resp.status_code} {resp.text}")
-            return 1
-        body = resp.json().get("body") or ""
-        if done_marker in body:
-            print(f"Marker {done_marker} already present; nothing to do.")
-            return 0
-        if args.marker in body:
-            break
-        print(
-            f"Marker {args.marker} not found "
-            f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry."
-        )
-    else:
-        print(
-            f"WARNING: marker {args.marker} not found after "
-            f"{len(RETRY_DELAYS_SEC)} attempts; skipping writeback. "
-            f"The handler may have failed to edit the placeholder comment."
-        )
-        return 0
-
-    new_lines = []
-    for line in body.splitlines(keepends=True):
-        if args.marker in line:
-            line = line.replace("⏳", icon, 1)
-            line = line.replace(args.marker, done_marker)
-        new_lines.append(line)
-    new_body = "".join(new_lines)
-
-    patch = requests.patch(
-        url,
-        headers=headers,
-        json={"body": new_body},
-        timeout=15,
-    )
-    if patch.status_code != 200:
-        print(f"PATCH failed: {patch.status_code} {patch.text}")
-        return 1
-    print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())