ci: add /rerun-group to rerun all registered tests in a group (#24023)

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

.claude/skills/ci-workflow-guide/SKILL.md

@@ -12,6 +12,7 @@ This skill covers the CI **infrastructure** layer — how tests are dispatched,
 ## Naming Conventions
 
 - **Suite**: `stage-{a,b,c}-test-{gpu_count}-gpu-{hardware}` (e.g., `stage-b-test-1-gpu-small`)
+- **Test group**: Directory-level registered test group under `test/registered/` (e.g., `hicache` maps to `test/registered/hicache/test_*.py`)
 - **CI runner**: `{gpu_count}-gpu-{hardware}` (e.g., `1-gpu-5090`, `4-gpu-h100`, `8-gpu-h200`)
 
 ---
@@ -385,5 +386,6 @@ group: pr-test-{event_name}-{branch}-{pr_sha}-{stage}
 | `/tag-and-rerun-ci` | Adds label + reruns |
 | `/rerun-stage <stage>` | Dispatches `pr-test.yml` with `target_stage=<stage>` |
 | `/rerun-test <test-file>` | Reruns a specific test file via `rerun-test.yml` |
+| `/rerun-group <group> [<group> ...]` | Expands registered test groups, then reuses `/rerun-test` |
 
 Handled by `scripts/ci/utils/slash_command_handler.py` → `.github/workflows/slash-command-handler.yml`.

.github/workflows/slash-command-handler.yml

@@ -20,6 +20,7 @@ jobs:
        contains(github.event.comment.body, '/rerun-failed-ci') ||
        contains(github.event.comment.body, '/tag-and-rerun-ci') ||
        contains(github.event.comment.body, '/rerun-stage') ||
+       contains(github.event.comment.body, '/rerun-group') ||
        contains(github.event.comment.body, '/rerun-test'))
     runs-on: ubuntu-latest
 

scripts/ci/utils/slash_command_handler.py

@@ -523,6 +523,49 @@ MULTIMODAL_PATH_TO_RUNNER = {
 MULTIMODAL_DEFAULT_RUNNER = "1-gpu-h100"
 
 
+def _known_test_groups():
+    groups = []
+    for group_dir in glob.glob("test/registered/*"):
+        if os.path.isdir(group_dir):
+            groups.append(os.path.basename(group_dir))
+    return sorted(groups)
+
+
+def resolve_test_group_specs(group_name):
+    """
+    Resolve a test group name into /rerun-test specs.
+
+    A group maps to a directory under test/registered/. For example,
+    "hicache" maps to all test_*.py files under test/registered/hicache/.
+
+    Returns (test_specs, error_message). On success error_message is None.
+    """
+    group_name = group_name.strip().strip("/")
+    if (
+        not group_name
+        or group_name.startswith(".")
+        or "/." in group_name
+        or ".." in group_name.split("/")
+    ):
+        return [], f"Invalid test group `{group_name}`."
+
+    group_dir = os.path.join("test", "registered", group_name)
+    if not os.path.isdir(group_dir):
+        known = ", ".join(f"`{g}`" for g in _known_test_groups())
+        return (
+            [],
+            f"Unknown test group `{group_name}`.\n\nKnown groups: {known}",
+        )
+
+    test_files = sorted(
+        glob.glob(os.path.join(group_dir, "**", "test_*.py"), recursive=True)
+    )
+    if not test_files:
+        return [], f"No registered test files found in `{group_dir}`."
+
+    return [os.path.relpath(path, "test") for path in test_files], None
+
+
 def resolve_test_file(file_part):
     """
     Resolve a user-provided file path to a path relative to test/ or full path for multimodal.
@@ -842,23 +885,21 @@ def _dispatch_batch(gh_repo, pr, batch, token):
         }
 
 
-def handle_rerun_test(gh_repo, pr, comment, user_perms, test_specs, token):
+def _check_rerun_test_permissions(gh_repo, pr, comment, user_perms, command_name):
     """
-    Handles the /rerun-test command. Resolves all test specs, groups them by
+    Check permissions shared by /rerun-test and /rerun-group.
-    (runner_label, use_deepep, is_cpu), and dispatches one workflow per group.
     """
-    # SECURITY: For fork PRs, only allow /rerun-test if the commenter has write+ permission.
+    # SECURITY: These commands check out and execute code from the PR branch on
-    # This command checks out and executes code from the PR branch on self-hosted GPU
+    # self-hosted GPU runners, so fork PRs require a trusted collaborator.
-    # runners, so we must ensure the commenter is a trusted collaborator.
     is_fork = pr.head.repo is None or pr.head.repo.owner.login != gh_repo.owner.login
     if is_fork:
         commenter = comment.user.login
         perm = gh_repo.get_collaborator_permission(commenter)
         if perm not in ("admin", "write"):
-            print(f"Permission denied: /rerun-test on fork PR by {commenter}.")
+            print(f"Permission denied: /{command_name} on fork PR by {commenter}.")
             comment.create_reaction("confused")
             pr.create_issue_comment(
-                "❌ `/rerun-test` is not available for fork PRs unless the commenter "
+                f"❌ `/{command_name}` is not available for fork PRs unless the commenter "
                 "has write permission on the repo.\n\n"
                 "Please ask a maintainer to run this command, or use the normal CI flow."
             )
@@ -872,6 +913,21 @@ def handle_rerun_test(gh_repo, pr, comment, user_perms, test_specs, token):
         print("Permission denied: neither can_rerun_test nor can_rerun_stage is true.")
         return False
 
+    return True
+
+
+def handle_rerun_test(
+    gh_repo, pr, comment, user_perms, test_specs, token, skip_permission_check=False
+):
+    """
+    Handles the /rerun-test command. Resolves all test specs, groups them by
+    (runner_label, use_deepep, is_cpu), and dispatches one workflow per group.
+    """
+    if not skip_permission_check and not _check_rerun_test_permissions(
+        gh_repo, pr, comment, user_perms, "rerun-test"
+    ):
+        return False
+
     if not test_specs:
         comment.create_reaction("confused")
         pr.create_issue_comment(
@@ -959,6 +1015,56 @@ def handle_rerun_test(gh_repo, pr, comment, user_perms, test_specs, token):
     return len(successes) > 0
 
 
+def handle_rerun_group(gh_repo, pr, comment, user_perms, group_names, token):
+    """
+    Handles the /rerun-group command. Expands one or more registered test
+    groups into test file specs, then reuses /rerun-test dispatch behavior.
+    """
+    if not _check_rerun_test_permissions(
+        gh_repo, pr, comment, user_perms, "rerun-group"
+    ):
+        return False
+
+    if not group_names:
+        comment.create_reaction("confused")
+        pr.create_issue_comment(
+            "❌ Please specify a test group: `/rerun-group <group>`\n\n"
+            "Example:\n"
+            "- `/rerun-group hicache`"
+        )
+        return False
+
+    test_specs = []
+    failures = []
+    seen = set()
+    for group_name in group_names:
+        specs, err = resolve_test_group_specs(group_name)
+        if err:
+            failures.append((group_name, err))
+            continue
+
+        for spec in specs:
+            if spec not in seen:
+                test_specs.append(spec)
+                seen.add(spec)
+
+    if failures:
+        comment.create_reaction("confused")
+        lines = [f"❌ `{group}`: {err}" for group, err in failures]
+        pr.create_issue_comment("\n\n".join(lines))
+        return False
+
+    return handle_rerun_test(
+        gh_repo,
+        pr,
+        comment,
+        user_perms,
+        test_specs,
+        token,
+        skip_permission_check=True,
+    )
+
+
 def main():
     # 1. Load Environment Variables
     token = get_env_var("GITHUB_TOKEN")
@@ -1040,6 +1146,10 @@ def main():
         stage_name = parts[1].strip() if len(parts) > 1 else None
         handle_rerun_stage(repo, pr, comment, user_perms, stage_name, token)
 
+    elif first_line.startswith("/rerun-group"):
+        group_names = first_line.split()[1:]
+        handle_rerun_group(repo, pr, comment, user_perms, group_names or None, token)
+
     elif first_line.startswith("/rerun-test"):
         test_specs = first_line.split()[1:]
         handle_rerun_test(repo, pr, comment, user_perms, test_specs or None, token)