From ba625d529060264fdb176dae4b5ef8823e2cb9f9 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sat, 9 May 2026 03:19:24 -0700 Subject: [PATCH] slash command rerun UX: emoji semantics + result writeback (#24802) --- .github/workflows/rerun-test.yml | 47 +++++++++ scripts/ci/utils/slash_command_handler.py | 64 ++++++++---- scripts/ci/utils/write_rerun_test_result.py | 104 ++++++++++++++++++++ 3 files changed, 194 insertions(+), 21 deletions(-) create mode 100755 scripts/ci/utils/write_rerun_test_result.py diff --git a/.github/workflows/rerun-test.yml b/.github/workflows/rerun-test.yml index 1241763f3..f84f39c93 100644 --- a/.github/workflows/rerun-test.yml +++ b/.github/workflows/rerun-test.yml @@ -44,6 +44,16 @@ on: required: false type: string default: "false" + reply_comment_id: + description: "Reply comment ID to write back result to" + required: false + type: string + default: "" + reply_marker: + description: "Per-batch marker for locating the line in reply comment" + required: false + type: string + default: "" env: SGLANG_IS_IN_CI: true @@ -196,3 +206,40 @@ jobs: done total_elapsed=$(( SECONDS - suite_start )) echo "All $total test(s) passed in ${total_elapsed}s" + + write-back-result: + needs: [rerun-test-cuda, rerun-test-cpu] + if: always() && inputs.reply_comment_id != '' && inputs.reply_marker != '' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + concurrency: + group: rerun-test-writeback-${{ inputs.reply_comment_id }} + cancel-in-progress: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: pip install requests + + - name: Write back result to reply comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [[ "${{ needs.rerun-test-cuda.result }}" == "success" || "${{ needs.rerun-test-cpu.result }}" == "success" ]]; then + STATUS=success + else + STATUS=failure + fi + python3 scripts/ci/utils/write_rerun_test_result.py \ + --comment-id "${{ inputs.reply_comment_id }}" \ + --marker "${{ inputs.reply_marker }}" \ + --status "$STATUS" \ + --repo "${{ github.repository }}" diff --git a/scripts/ci/utils/slash_command_handler.py b/scripts/ci/utils/slash_command_handler.py index 0f863597e..15e25be56 100644 --- a/scripts/ci/utils/slash_command_handler.py +++ b/scripts/ci/utils/slash_command_handler.py @@ -404,7 +404,7 @@ def handle_rerun_stage( print("Error: No stage name provided") comment.create_reaction("confused") pr.create_issue_comment( - f"❌ Please specify a stage name: `/rerun-stage `\n\n" + f"⛔ Please specify a stage name: `/rerun-stage `\n\n" f"Examples: `/rerun-stage unit-test-backend-4-gpu`, `/rerun-stage accuracy-test-1-gpu`" ) return False @@ -456,7 +456,7 @@ def handle_rerun_stage( if stage_name not in valid_stages: comment.create_reaction("confused") pr.create_issue_comment( - f"❌ Stage `{stage_name}` doesn't support isolated runs yet.\n\n" + f"⛔ Stage `{stage_name}` doesn't support isolated runs yet.\n\n" f"**NVIDIA stages:**\n" + "\n".join(f"- `{s}`" for s in nvidia_stages) + "\n\n**AMD stages:**\n" @@ -573,13 +573,13 @@ def handle_rerun_stage( ) if run_url: pr.create_issue_comment( - f"✅ Triggered `{stage_name}` to run independently" + f"🚀 Triggered `{stage_name}` to run independently" f" (skipping dependencies)." f" [View workflow run]({run_url})" ) else: pr.create_issue_comment( - f"✅ Triggered `{stage_name}` to run independently" + f"🚀 Triggered `{stage_name}` to run independently" f" (skipping dependencies).\n" f"⚠️ Could not retrieve workflow run URL. " f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress." @@ -593,7 +593,7 @@ def handle_rerun_stage( print(f"Error triggering workflow_dispatch: {e}") comment.create_reaction("confused") pr.create_issue_comment( - f"❌ Failed to trigger workflow: {str(e)}\n\n" + f"⛔ Failed to trigger workflow: {str(e)}\n\n" f"Please check the logs or contact maintainers." ) return False @@ -908,7 +908,7 @@ def _resolve_test_spec(test_spec): } -def _dispatch_batch(gh_repo, pr, batch, token): +def _dispatch_batch(gh_repo, pr, batch, token, reply_comment_id="", reply_marker=""): """ Dispatch a single workflow run for a batch of resolved test specs that share the same (runner_label, use_deepep, is_cpu). @@ -951,6 +951,8 @@ def _dispatch_batch(gh_repo, pr, batch, token): "use_deepep": str(use_deepep).lower(), "is_cpu": str(is_cpu).lower(), "install_diffusion": str(install_diffusion).lower(), + "reply_comment_id": str(reply_comment_id) if reply_comment_id else "", + "reply_marker": reply_marker, } if is_fork: ref = "main" @@ -998,6 +1000,7 @@ def _dispatch_batch(gh_repo, pr, batch, token): "test_commands": test_commands, "runner_label": runner_label, "run_url": run_url, + "reply_marker": reply_marker, } except Exception as e: @@ -1023,7 +1026,7 @@ def _check_rerun_test_permissions(gh_repo, pr, comment, user_perms, command_name print(f"Permission denied: /{command_name} on fork PR by {commenter}.") comment.create_reaction("confused") pr.create_issue_comment( - f"❌ `/{command_name}` is not available for fork PRs unless the commenter " + f"⛔ `/{command_name}` is not available for fork PRs unless the commenter " "has write permission on the repo.\n\n" "Please ask a maintainer to run this command, or use the normal CI flow." ) @@ -1055,7 +1058,7 @@ def handle_rerun_test( if not test_specs: comment.create_reaction("confused") pr.create_issue_comment( - "❌ Please specify a test: `/rerun-test ::`\n\n" + "⛔ Please specify a test: `/rerun-test ::`\n\n" "Examples:\n" "- `/rerun-test test/registered/core/test_srt_endpoint.py::TestSRTEndpoint.test_simple_decode`\n" "- `/rerun-test registered/core/test_srt_endpoint.py::TestSRTEndpoint`\n" @@ -1091,12 +1094,30 @@ def handle_rerun_test( ) groups.setdefault(key, []).append(r) - # Phase 3: Dispatch one workflow per group - dispatch_results = [] - for batch in groups.values(): - dispatch_results.append(_dispatch_batch(gh_repo, pr, batch, token)) + # Phase 3a: Create placeholder reply comment so we have its ID before + # dispatching workflows. This lets each dispatched run write its + # success/failure result back to the right line in this comment. + reply_comment = pr.create_issue_comment("🚀 Dispatching rerun-test workflow(s)...") - # Build consolidated comment + # Phase 3b: Dispatch one workflow per group, with a unique per-batch + # marker each. The marker is an HTML comment that the writeback step + # uses to locate the line and replace 🚀 with ✅/❌. + dispatch_results = [] + for idx, batch in enumerate(groups.values()): + marker = f"" + dispatch_results.append( + _dispatch_batch( + gh_repo, + pr, + batch, + token, + reply_comment_id=reply_comment.id, + reply_marker=marker, + ) + ) + + # Build consolidated comment body (markers placed at line ends so the + # writeback step can locate and update each line). lines = [] for dr in dispatch_results: if dr["success"]: @@ -1113,25 +1134,26 @@ def handle_rerun_test( cmds = "\n".join( f"cd test/ && python3 {cmd}" for cmd in dr["test_commands"] ) + marker = dr.get("reply_marker", "") if dr.get("run_url"): lines.append( - f"✅ `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): " - f"[View workflow run]({dr['run_url']})\n" + f"🚀 `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): " + f"⏳ [View workflow run]({dr['run_url']}) {marker}\n" f"```\n{cmds}\n```" ) else: lines.append( - f"✅ `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}):\n" + f"🚀 `{dr['runner_label']}` ({len(dr['test_commands'])} test{'s' if len(dr['test_commands']) > 1 else ''}): ⏳ {marker}\n" f"```\n{cmds}\n```\n" f"⚠️ Could not retrieve workflow run URL. " f"Check the [Actions tab](https://github.com/{gh_repo.full_name}/actions) for progress." ) else: specs_str = ", ".join(f"`{s}`" for s in dr["specs"]) - lines.append(f"❌ {specs_str}: {dr['error']}") + lines.append(f"⛔ {specs_str}: {dr['error']}") for r in resolve_failures: - lines.append(f"❌ `{r['spec']}`: {r['error']}") + lines.append(f"⛔ `{r['spec']}`: {r['error']}") body = "\n\n".join(lines) @@ -1141,7 +1163,7 @@ def handle_rerun_test( if not successes and (resolve_failures or dispatch_results): comment.create_reaction("confused") - pr.create_issue_comment(body) + reply_comment.edit(body) return len(successes) > 0 @@ -1158,7 +1180,7 @@ def handle_rerun_group(gh_repo, pr, comment, user_perms, group_names, token): if not group_names: comment.create_reaction("confused") pr.create_issue_comment( - "❌ Please specify a test group: `/rerun-group `\n\n" + "⛔ Please specify a test group: `/rerun-group `\n\n" "Example:\n" "- `/rerun-group hicache`" ) @@ -1180,7 +1202,7 @@ def handle_rerun_group(gh_repo, pr, comment, user_perms, group_names, token): if failures: comment.create_reaction("confused") - lines = [f"❌ `{group}`: {err}" for group, err in failures] + lines = [f"⛔ `{group}`: {err}" for group, err in failures] pr.create_issue_comment("\n\n".join(lines)) return False diff --git a/scripts/ci/utils/write_rerun_test_result.py b/scripts/ci/utils/write_rerun_test_result.py new file mode 100755 index 000000000..a3f06b9b7 --- /dev/null +++ b/scripts/ci/utils/write_rerun_test_result.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +Write back the result of a Rerun Test workflow run into its parent reply +comment, by replacing the hourglass placeholder on the marker's line with +a success/failure icon. The leading rocket on the line is preserved as a +visual anchor that this line came from a slash-command trigger. + +Concurrent dispatches against the same reply comment are serialized at the +GitHub Actions layer via job-level `concurrency`, so a simple +read-modify-write here is sufficient. Marker is rewritten to a `:done` +variant to make the operation idempotent against accidental reruns. +""" + +import argparse +import os +import sys +import time + +import requests + +# When the marker isn't found, retry a few times before giving up. The +# usual cause is a brief race where the dispatched workflow finishes its +# writeback step before the handler edits the placeholder comment with the +# final body containing markers. If retries don't help (e.g. the handler +# failed to edit the placeholder at all), we still return 0 — failing here +# would amplify a single placeholder-edit failure into N noisy finalizer +# job failures, one per dispatched batch. +RETRY_DELAYS_SEC = [0, 5, 15] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--comment-id", required=True, type=int) + ap.add_argument( + "--marker", required=True, help="Per-batch marker, e.g. " + ) + ap.add_argument("--status", required=True, choices=["success", "failure"]) + ap.add_argument("--repo", required=True, help="owner/repo") + args = ap.parse_args() + + token = os.environ.get("GITHUB_TOKEN") + if not token: + print("ERROR: GITHUB_TOKEN not set") + return 1 + + icon = "✅" if args.status == "success" else "❌" + done_marker = args.marker.replace("-->", ":done-->") + + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + url = f"https://api.github.com/repos/{args.repo}/issues/comments/{args.comment_id}" + + body = None + for attempt, delay in enumerate(RETRY_DELAYS_SEC): + if delay: + time.sleep(delay) + resp = requests.get(url, headers=headers, timeout=15) + if resp.status_code != 200: + print(f"GET failed: {resp.status_code} {resp.text}") + return 1 + body = resp.json().get("body") or "" + if done_marker in body: + print(f"Marker {done_marker} already present; nothing to do.") + return 0 + if args.marker in body: + break + print( + f"Marker {args.marker} not found " + f"(attempt {attempt + 1}/{len(RETRY_DELAYS_SEC)}); will retry." + ) + else: + print( + f"WARNING: marker {args.marker} not found after " + f"{len(RETRY_DELAYS_SEC)} attempts; skipping writeback. " + f"The handler may have failed to edit the placeholder comment." + ) + return 0 + + new_lines = [] + for line in body.splitlines(keepends=True): + if args.marker in line: + line = line.replace("⏳", icon, 1) + line = line.replace(args.marker, done_marker) + new_lines.append(line) + new_body = "".join(new_lines) + + patch = requests.patch( + url, + headers=headers, + json={"body": new_body}, + timeout=15, + ) + if patch.status_code != 200: + print(f"PATCH failed: {patch.status_code} {patch.text}") + return 1 + print(f"Updated comment {args.comment_id}: {args.marker} -> {icon}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())