slash command rerun UX: emoji semantics + result writeback (#24802)

.github/workflows/rerun-test.yml

@@ -44,6 +44,16 @@ on:
         required: false
         type: string
         default: "false"
+      reply_comment_id:
+        description: "Reply comment ID to write back result to"
+        required: false
+        type: string
+        default: ""
+      reply_marker:
+        description: "Per-batch marker for locating the line in reply comment"
+        required: false
+        type: string
+        default: ""
 
 env:
   SGLANG_IS_IN_CI: true
@@ -196,3 +206,40 @@ jobs:
           done
           total_elapsed=$(( SECONDS - suite_start ))
           echo "All $total test(s) passed in ${total_elapsed}s"
+
+  write-back-result:
+    needs: [rerun-test-cuda, rerun-test-cpu]
+    if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != ''
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    concurrency:
+      group: rerun-test-writeback-${{ inputs.reply_comment_id }}
+      cancel-in-progress: false
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: pip install requests
+
+      - name: Write back result to reply comment
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ "${{ needs.rerun-test-cuda.result }}" == "success" || "${{ needs.rerun-test-cpu.result }}" == "success" ]]; then
+            STATUS=success
+          else
+            STATUS=failure
+          fi
+          python3 scripts/ci/utils/write_rerun_test_result.py \
+            --comment-id "${{ inputs.reply_comment_id }}" \
+            --marker "${{ inputs.reply_marker }}" \
+            --status "$STATUS" \
+            --repo "${{ github.repository }}"

scripts/ci/utils/slash_command_handler.py

@@ -404,7 +404,7 @@ def handle_rerun_stage(
         print("Error: No stage name provided")
         comment.create_reaction("confused")
         pr.create_issue_comment(
-            f"❌ Please specify a stage name: `/rerun-stage <stage-name>`\n\n"
+            f"⛔ Please specify a stage name: `/rerun-stage <stage-name>`\n\n"
             f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`"
         )
         return False
@@ -456,7 +456,7 @@ def handle_rerun_stage(
     if stage_name not in valid_stages:
         comment.create_reaction("confused")
         pr.create_issue_comment(
-            f"❌ Stage `{stage_name}` doesn't support isolated runs yet.\n\n"
+            f"⛔ Stage `{stage_name}` doesn't support isolated runs yet.\n\n"
             f"**NVIDIA stages:**\n"
             + "\n".join(f"- `{s}`" for s in nvidia_stages)
             + "\n\n**AMD stages:**\n"
@@ -573,13 +573,13 @@ def handle_rerun_stage(
                 )
                 if run_url:
                     pr.create_issue_comment(
-                        f"✅ Triggered `{stage_name}` to run independently"
+                        f"🚀 Triggered `{stage_name}` to run independently"
                         f" (skipping dependencies)."
                         f" [View workflow run]({run_url})"
                     )
                 else:
                     pr.create_issue_comment(
-                        f"✅ Triggered `{stage_name}` to run independently"
+                        f"🚀 Triggered `{stage_name}` to run independently"
                         f" (skipping dependencies).\n"
                         f"⚠️ Could not retrieve workflow run URL. "
                         f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
@@ -593,7 +593,7 @@ def handle_rerun_stage(
         print(f"Error triggering workflow_dispatch: {e}")
         comment.create_reaction("confused")
         pr.create_issue_comment(
-            f"❌ Failed to trigger workflow: {str(e)}\n\n"
+            f"⛔ Failed to trigger workflow: {str(e)}\n\n"
             f"Please check the logs or contact maintainers."
         )
         return False
@@ -908,7 +908,7 @@ def _resolve_test_spec(test_spec):
     }
 
 
-def _dispatch_batch(gh_repo, pr, batch, token):
+def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker=""):
     """
     Dispatch a single workflow run for a batch of resolved test specs
     that share the same (runner_label, use_deepep, is_cpu).
@@ -951,6 +951,8 @@ def _dispatch_batch(gh_repo, pr, batch, token):
             "use_deepep": str(use_deepep).lower(),
             "is_cpu": str(is_cpu).lower(),
             "install_diffusion": str(install_diffusion).lower(),
+            "reply_comment_id": str(reply_comment_id) if reply_comment_id else "",
+            "reply_marker": reply_marker,
         }
         if is_fork:
             ref = "main"
@@ -998,6 +1000,7 @@ def _dispatch_batch(gh_repo, pr, batch, token):
             "test_commands": test_commands,
             "runner_label": runner_label,
             "run_url": run_url,
+            "reply_marker": reply_marker,
         }
 
     except Exception as e:
@@ -1023,7 +1026,7 @@ def _check_rerun_test_permissions(gh_repo, pr, comment, user_perms, command_name
             print(f"Permission denied: /{command_name} on fork PR by {commenter}.")
             comment.create_reaction("confused")
             pr.create_issue_comment(
-                f"❌ `/{command_name}` is not available for fork PRs unless the commenter "
+                f"⛔ `/{command_name}` is not available for fork PRs unless the commenter "
                 "has write permission on the repo.\n\n"
                 "Please ask a maintainer to run this command, or use the normal CI flow."
             )
@@ -1055,7 +1058,7 @@ def handle_rerun_test(
     if not test_specs:
         comment.create_reaction("confused")
         pr.create_issue_comment(
-            "❌ Please specify a test: `/rerun-test <file>::<TestClass.test_method>`\n\n"
+            "⛔ Please specify a test: `/rerun-test <file>::<TestClass.test_method>`\n\n"
             "Examples:\n"
             "- `/rerun-test test/registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`\n"
             "- `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint`\n"
@@ -1091,12 +1094,30 @@ def handle_rerun_test(
         )
         groups.setdefault(key, []).append(r)
 
-    # Phase 3: Dispatch one workflow per group
+    # Phase 3a: Create placeholder reply comment so we have its ID before
-    dispatch_results = []
+    # dispatching workflows. This lets each dispatched run write its
-    for batch in groups.values():
+    # success/failure result back to the right line in this comment.
-        dispatch_results.append(_dispatch_batch(gh_repo, pr, batch, token))
+    reply_comment = pr.create_issue_comment("🚀 Dispatching rerun-test workflow(s)...")
 
-    # Build consolidated comment
+    # Phase 3b: Dispatch one workflow per group, with a unique per-batch
+    # marker each. The marker is an HTML comment that the writeback step
+    # uses to locate the line and replace 🚀 with ✅/❌.
+    dispatch_results = []
+    for idx, batch in enumerate(groups.values()):
+        marker = f"<!--rrt:{idx}-->"
+        dispatch_results.append(
+            _dispatch_batch(
+                gh_repo,
+                pr,
+                batch,
+                token,
+                reply_comment_id=reply_comment.id,
+                reply_marker=marker,
+            )
+        )
+
+    # Build consolidated comment body (markers placed at line ends so the
+    # writeback step can locate and update each line).
     lines = []
     for dr in dispatch_results:
         if dr["success"]:
@@ -1113,25 +1134,26 @@ def handle_rerun_test(
                 cmds = "\n".join(
                     f"cd test/ && python3 {cmd}" for cmd in dr["test_commands"]
                 )
+            marker = dr.get("reply_marker", "")
             if dr.get("run_url"):
                 lines.append(
-                    f"✅ `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): "
+                    f"🚀 `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): "
-                    f"[View workflow run]({dr['run_url']})\n"
+                    f"⏳ [View workflow run]({dr['run_url']}) {marker}\n"
                     f"```\n{cmds}\n```"
                 )
             else:
                 lines.append(
-                    f"✅ `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}):\n"
+                    f"🚀 `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): ⏳ {marker}\n"
                     f"```\n{cmds}\n```\n"
                     f"⚠️ Could not retrieve workflow run URL. "
                     f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress."
                 )
         else:
             specs_str = ", ".join(f"`{s}`" for s in dr["specs"])
-            lines.append(f"❌ {specs_str}: {dr['error']}")
+            lines.append(f"⛔ {specs_str}: {dr['error']}")
 
     for r in resolve_failures:
-        lines.append(f"❌ `{r['spec']}`: {r['error']}")
+        lines.append(f"⛔ `{r['spec']}`: {r['error']}")
 
     body = "\n\n".join(lines)
 
@@ -1141,7 +1163,7 @@ def handle_rerun_test(
     if not successes and (resolve_failures or dispatch_results):
         comment.create_reaction("confused")
 
-    pr.create_issue_comment(body)
+    reply_comment.edit(body)
     return len(successes) > 0
 
 
@@ -1158,7 +1180,7 @@ def handle_rerun_group(gh_repo, pr, comment, user_perms, group_names, token):
     if not group_names:
         comment.create_reaction("confused")
         pr.create_issue_comment(
-            "❌ Please specify a test group: `/rerun-group <group>`\n\n"
+            "⛔ Please specify a test group: `/rerun-group <group>`\n\n"
             "Example:\n"
             "- `/rerun-group hicache`"
         )
@@ -1180,7 +1202,7 @@ def handle_rerun_group(gh_repo, pr, comment, user_perms, group_names, token):
 
     if failures:
         comment.create_reaction("confused")
-        lines = [f"❌ `{group}`: {err}" for group, err in failures]
+        lines = [f"⛔ `{group}`: {err}" for group, err in failures]
         pr.create_issue_comment("\n\n".join(lines))
         return False
 

scripts/ci/utils/write_rerun_test_result.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Write back the result of a Rerun Test workflow run into its parent reply
+comment, by replacing the hourglass placeholder on the marker's line with
+a success/failure icon. The leading rocket on the line is preserved as a
+visual anchor that this line came from a slash-command trigger.
+
+Concurrent dispatches against the same reply comment are serialized at the
+GitHub Actions layer via job-level `concurrency`, so a simple
+read-modify-write here is sufficient. Marker is rewritten to a `:done`
+variant to make the operation idempotent against accidental reruns.
+"""
+
+import argparse
+import os
+import sys
+import time
+
+import requests
+
+# When the marker isn't found, retry a few times before giving up. The
+# usual cause is a brief race where the dispatched workflow finishes its
+# writeback step before the handler edits the placeholder comment with the
+# final body containing markers. If retries don't help (e.g. the handler
+# failed to edit the placeholder at all), we still return 0 — failing here
+# would amplify a single placeholder-edit failure into N noisy finalizer
+# job failures, one per dispatched batch.
+RETRY_DELAYS_SEC = [0, 5, 15]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--comment-id", required=True, type=int)
+    ap.add_argument(
+        "--marker", required=True, help="Per-batch marker, e.g. <!--rrt:0-->"
+    )
+    ap.add_argument("--status", required=True, choices=["success", "failure"])
+    ap.add_argument("--repo", required=True, help="owner/repo")
+    args = ap.parse_args()
+
+    token = os.environ.get("GITHUB_TOKEN")
+    if not token:
+        print("ERROR: GITHUB_TOKEN not set")
+        return 1
+
+    icon = "✅" if args.status == "success" else "❌"
+    done_marker = args.marker.replace("-->", ":done-->")
+
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}"
+
+    body = None
+    for attempt, delay in enumerate(RETRY_DELAYS_SEC):
+        if delay:
+            time.sleep(delay)
+        resp = requests.get(url, headers=headers, timeout=15)
+        if resp.status_code != 200:
+            print(f"GET failed: {resp.status_code} {resp.text}")
+            return 1
+        body = resp.json().get("body") or ""
+        if done_marker in body:
+            print(f"Marker {done_marker} already present; nothing to do.")
+            return 0
+        if args.marker in body:
+            break
+        print(
+            f"Marker {args.marker} not found "
+            f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry."
+        )
+    else:
+        print(
+            f"WARNING: marker {args.marker} not found after "
+            f"{len(RETRY_DELAYS_SEC)} attempts; skipping writeback. "
+            f"The handler may have failed to edit the placeholder comment."
+        )
+        return 0
+
+    new_lines = []
+    for line in body.splitlines(keepends=True):
+        if args.marker in line:
+            line = line.replace("⏳", icon, 1)
+            line = line.replace(args.marker, done_marker)
+        new_lines.append(line)
+    new_body = "".join(new_lines)
+
+    patch = requests.patch(
+        url,
+        headers=headers,
+        json={"body": new_body},
+        timeout=15,
+    )
+    if patch.status_code != 200:
+        print(f"PATCH failed: {patch.status_code} {patch.text}")
+        return 1
+    print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())