[CI] Split the CI control labels into four axes and resolve them live (#40527)

.github/MAINTAINER.md

@@ -143,6 +143,17 @@ This section lists the oncalls for each hardware platform. The format is @github
 
 This list is based on the current situation. If you or someone you know would like to donate machines for CI, they can serve as the CI oncalls for their machines. Please ping [Lianmin Zheng](https://github.com/merrymercy) and [Ying Sheng](https://github.com/Ying1123) in the Slack channel. They will start a nomination and internal review process.
 
+## CI Control Labels
+
+`bypass-fail-fast`, `parallel-stages`, `max-concurrency` and `highest-priority`
+each relax one of the limits that keep a single PR from monopolizing the
+self-hosted GPU runners; `highest-priority` relaxes all of them at once. The
+[contribution guide](https://docs.sglang.io/developer_guide/contribution_guide.html#ci-control-labels)
+describes what each one does.
+
+Applying one spends other PRs' runner capacity. `parallel-stages` is the
+expensive one: a PR that cannot pass now runs its whole matrix.
+
 ## CI Maintenance Mode
 When the CI is unhealthy (e.g., the scheduled pr-test on `main` is broken for consecutive runs), the project enters **CI Maintenance Mode** by opening [issue #21065](https://github.com/sgl-project/sglang/issues/21065). While active:
 - All PR CI runs are paused. Resources are allocated to PRs that fix the CI.

.github/actions/check-pr-test-health/action.test.cjs

@@ -6,8 +6,10 @@ const { test } = require('node:test');
 const yaml = fs.readFileSync(path.join(__dirname, 'action.yml'), 'utf8');
 const script = yaml.split('        script: |\n')[1]
   .split('\n').map(line => line.replace(/^          /, '')).join('\n');
+// `require` and GITHUB_WORKSPACE mirror what actions/github-script injects.
+const WORKSPACE = path.join(__dirname, '..', '..', '..');
 const run = new (Object.getPrototypeOf(async function () {}).constructor)(
-  'github', 'context', 'core', 'process', script,
+  'github', 'context', 'core', 'process', 'require', script,
 );
 
 async function check({ snapshot = [], live = [], lint = 'success', event = 'pull_request' } = {}) {
@@ -41,29 +43,30 @@ async function check({ snapshot = [], live = [], lint = 'success', event = 'pull
     payload: event === 'pull_request'
       ? { pull_request: { number: 42, head: { sha: 'head' }, labels: labels(snapshot) } }
       : {},
-  }, { info: () => {}, setFailed: message => failures.push(message) }, { env: {} });
+  }, { info: () => {}, setFailed: message => failures.push(message) },
+     { env: { GITHUB_WORKSPACE: WORKSPACE } }, require);
   return { failures, labelReads, jobReads };
 }
 
 test('a label added after the event bypasses sibling failures on rerun', async () => {
-  assert.deepEqual(await check({ live: ['bypass-fastfail'] }),
+  assert.deepEqual(await check({ live: ['bypass-fail-fast'] }),
     { failures: [], labelReads: 1, jobReads: 0 });
 });
 
 test('a removed label does not continue bypassing sibling failures', async () => {
-  const result = await check({ snapshot: ['bypass-fastfail'] });
+  const result = await check({ snapshot: ['bypass-fail-fast'] });
   assert.equal(result.labelReads, 1);
   assert.equal(result.jobReads, 1);
   assert.match(result.failures[0], /root cause job\(s\): model-test/);
 });
 
 test('bypass never skips a failed lint check', async () => {
-  assert.deepEqual(await check({ live: ['bypass-fastfail'], lint: 'failure' }),
-    { failures: ['Fast-fail: lint check failed'], labelReads: 0, jobReads: 0 });
+  assert.deepEqual(await check({ live: ['bypass-fail-fast'], lint: 'failure' }),
+    { failures: ['Fail-fast: lint check failed'], labelReads: 0, jobReads: 0 });
 });
 
 test('non-PR events retain associated-PR label lookup', async () => {
-  assert.deepEqual(await check({ event: 'workflow_dispatch', live: ['bypass-fastfail'] }),
+  assert.deepEqual(await check({ event: 'workflow_dispatch', live: ['bypass-fail-fast'] }),
     { failures: [], labelReads: 1, jobReads: 0 });
 });
 

.github/actions/check-pr-test-health/action.yml

@@ -1,5 +1,5 @@
 name: Check PR Test Health
-description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fastfail` label.
+description: Fail fast if any job in the current workflow run has already failed, or if the lint check (from lint.yml) has failed. Auto-skips for scheduled runs. The jobs-failed check (but not the lint check) is bypassed when the PR carries the `bypass-fail-fast` label (or `highest-priority`, which implies it).
 
 inputs:
   github-token:
@@ -17,15 +17,17 @@ runs:
       with:
         github-token: ${{ inputs.github-token }}
         script: |
+          core.info(`[health-check] START -- event=${context.eventName}, runId=${context.runId}`);
+
           // Skip when explicitly requested via env var (e.g. release branch cut)
           if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
-            core.info('Skipping health check (SKIP_PR_TEST_HEALTH_CHECK=true)');
+            core.info('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
             return;
           }
 
-          // Skip for scheduled runs — they should collect all failures, not fast-fail
+          // Skip for scheduled runs -- they should collect all failures, not fail-fast
           if (context.eventName === 'schedule') {
-            core.info('Skipping health check for scheduled run');
+            core.info('[health-check] SKIP: scheduled run');
             return;
           }
 
@@ -33,6 +35,7 @@ runs:
           // listJobsForWorkflowRun only sees jobs within the SAME run, so we use
           // checks.listForRef which queries by commit SHA across ALL workflows.
           const ref = context.payload.pull_request?.head?.sha || context.sha;
+          core.info(`[health-check] Checking lint for ref=${ref}`);
           const { data } = await github.rest.checks.listForRef({
             owner: context.repo.owner,
             repo: context.repo.repo,
@@ -42,34 +45,20 @@ runs:
           const lintRun = data.check_runs.find(
             cr => cr.app?.slug === 'github-actions'
           );
+          core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
           if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
-            core.setFailed('Fast-fail: lint check failed');
+            core.setFailed('Fail-fast: lint check failed');
             return;
           }
 
-          // Skip the jobs-failed check when the PR carries the bypass-fastfail label.
-          // Lint check above still runs.
-          let labels = [];
-          if (context.payload.pull_request?.number) {
-            // reruns retain the original event payload, including stale labels
-            const { data: pr } = await github.rest.pulls.get({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              pull_number: context.payload.pull_request.number,
-            });
-            labels = pr.labels.map(l => l.name);
-          } else {
-            const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              commit_sha: ref,
-            });
-            if (prs.length > 0) {
-              labels = prs[0].labels.map(l => l.name);
-            }
-          }
-          if (labels.includes('bypass-fastfail')) {
-            core.info('Skipping jobs-failed check (bypass-fastfail label present)');
+          // The lint check above is never bypassed; only sibling failures are.
+          const { resolveCiLabels } = require(
+            `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
+          );
+          const axes = await resolveCiLabels(github, context);
+          core.info(`[health-check] PR labels: [${axes.labels.join(', ')}]`);
+          if (axes.bypassFailFast) {
+            core.info('[health-check] SKIP jobs-failed check: bypass-fail-fast label present');
             return;
           }
 
@@ -79,26 +68,41 @@ runs:
             run_id: context.runId,
             per_page: 100,
           });
-          // Find jobs that failed from a real error, not from fast-fail cascade
+          core.info(`[health-check] Total jobs in run: ${jobs.length}`);
+          const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
+          core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
+
+          // Find jobs that failed from a real error, not from fail-fast cascade
           const rootCauseFailures = jobs.filter(j => {
             if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
             // h20 runners are flaky (dirty GPU state from prior runs); their failures
-            // should not cascade fast-fail to other stages. j.name shape from
+            // should not cascade fail-fast to other stages. j.name shape from
             // listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
             // + optional " (<matrix>)". Split off the base job key before exact
             // match so we cover both inline + reusable forms without confusing
             // 'h20' with the 'h200' prefix.
             const baseName = j.name.split(/[ /]/)[0];
             if (baseName === 'base-c-test-8-gpu-h20') {
+              core.info(`[health-check] Filtered out h20 job: ${j.name}`);
               return false;
             }
-            // If the failing step is the health check, it's a cascade — skip it
+            // multimodal-gen NPU tests should not cascade fail-fast to perf/accuracy stages.
+            if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
+              core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
+              return false;
+            }
+            // If the failing step is the health check, it's a cascade -- skip it
             const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
             if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
+              core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
               return false;
             }
             return true;
           });
+          core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
+
           if (rootCauseFailures.length > 0) {
-            core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
+            core.setFailed(`Fail-fast: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
+          } else {
+            core.info('[health-check] PASS: no root cause failures detected');
           }

.github/actions/wait-for-jobs/action.yml

@@ -1,5 +1,5 @@
 name: Wait for Jobs
-description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `bypass-fastfail` label, letting downstream stages dispatch in parallel (same effect as scheduled runs).
+description: Poll and wait for specified jobs in the current workflow run to complete. Returns success immediately when the PR carries the `parallel-stages` label (or `highest-priority`, which implies it), letting downstream stages dispatch in parallel (same effect as scheduled runs).
 
 inputs:
   stage-name:
@@ -49,28 +49,12 @@ runs:
           const pollIntervalSeconds = parseInt(process.env.INPUT_POLL_INTERVAL_SECONDS);
           const maxAttempts = (maxWaitMinutes * 60) / pollIntervalSeconds;
 
-          // bypass-fastfail label opts the PR out of stage-to-stage waiting,
-          // letting all stages dispatch in parallel like scheduled runs do.
-          let labels = [];
-          if (context.payload.pull_request?.labels) {
-            labels = context.payload.pull_request.labels.map(l => l.name);
-          } else {
-            const ref = context.payload.pull_request?.head?.sha || context.sha;
-            try {
-              const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                commit_sha: ref,
-              });
-              if (prs.length > 0) {
-                labels = prs[0].labels.map(l => l.name);
-              }
-            } catch (e) {
-              console.log(`Could not fetch PR labels for ${ref}: ${e.message}`);
-            }
-          }
-          if (labels.includes('bypass-fastfail')) {
-            console.log(`Skipping ${stageName} wait (bypass-fastfail label present)`);
+          const { resolveCiLabels } = require(
+            `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
+          );
+          const { parallelStages } = await resolveCiLabels(github, context);
+          if (parallelStages) {
+            console.log(`Skipping ${stageName} wait (parallel-stages label present)`);
             core.setOutput('result', 'success');
             return;
           }

.github/scripts/ci-labels.cjs

@@ -0,0 +1,62 @@
+"use strict";
+
+/**
+ * Labels come from the API, not `context.payload`: a rerun replays the original
+ * event, so the payload carries the label set from when the run was created.
+ */
+
+const BYPASS_FAIL_FAST = "bypass-fail-fast";
+const PARALLEL_STAGES = "parallel-stages";
+const MAX_CONCURRENCY = "max-concurrency";
+const HIGHEST_PRIORITY = "highest-priority";
+
+// Callers gate the whole run on this, so a transient API error must not fail them.
+async function readLabels(github, context) {
+  try {
+    return await fetchLabels(github, context);
+  } catch (e) {
+    console.warn(`Could not read PR labels: ${e.message}`);
+    return [];
+  }
+}
+
+async function fetchLabels(github, context) {
+  const prNumber = context.payload.pull_request?.number;
+  if (prNumber) {
+    const { data: pr } = await github.rest.pulls.get({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      pull_number: prNumber,
+    });
+    return pr.labels.map((l) => l.name);
+  }
+  const sha = context.payload.pull_request?.head?.sha || context.sha;
+  const { data: prs } =
+    await github.rest.repos.listPullRequestsAssociatedWithCommit({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      commit_sha: sha,
+    });
+  return prs.length > 0 ? prs[0].labels.map((l) => l.name) : [];
+}
+
+async function resolveCiLabels(github, context) {
+  const labels = await readLabels(github, context);
+  const has = (name) => labels.includes(name);
+  const highestPriority = has(HIGHEST_PRIORITY);
+  return {
+    labels,
+    bypassFailFast: highestPriority || has(BYPASS_FAIL_FAST),
+    parallelStages: highestPriority || has(PARALLEL_STAGES),
+    maxConcurrency: highestPriority || has(MAX_CONCURRENCY),
+    highestPriority,
+  };
+}
+
+module.exports = {
+  resolveCiLabels,
+  BYPASS_FAIL_FAST,
+  PARALLEL_STAGES,
+  MAX_CONCURRENCY,
+  HIGHEST_PRIORITY,
+};

.github/workflows/_npu-pr-test-stage.yml

@@ -39,7 +39,7 @@ on:
         type: string
         default: ''
       skip_pr_test_health_check:
-        description: 'Set to true to skip the PR test health check (fast-fail gate).'
+        description: 'Set to true to skip the PR test health check (fail-fast gate).'
         type: string
         default: 'false'
       is_nightly_pipeline_job:
@@ -90,112 +90,7 @@ jobs:
         run: |
           git config --system --add safe.directory ${GITHUB_WORKSPACE}
 
-      - name: Check PR test health
-        uses: actions/github-script@v8
-        env:
-          SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
-        with:
-          github-token: ${{ inputs.github-token || github.token }}
-          script: |
-            core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
-
-            // Skip when explicitly requested via env var (e.g. release branch cut)
-            if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
-              core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
-              return;
-            }
-
-            // Skip for scheduled runs — they should collect all failures, not fast-fail
-            if (context.eventName === 'schedule') {
-              core.notice('[health-check] SKIP: scheduled run');
-              return;
-            }
-
-            // Check lint status from the separate Lint workflow (lint.yml).
-            // listJobsForWorkflowRun only sees jobs within the SAME run, so we use
-            // checks.listForRef which queries by commit SHA across ALL workflows.
-            const ref = context.payload.pull_request?.head?.sha || context.sha;
-            core.info(`[health-check] Checking lint for ref=${ref}`);
-            const { data } = await github.rest.checks.listForRef({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              ref: ref,
-              check_name: 'lint',
-            });
-            const lintRun = data.check_runs.find(
-              cr => cr.app?.slug === 'github-actions'
-            );
-            core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
-            if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
-              core.setFailed('Fast-fail: lint check failed');
-              return;
-            }
-
-            // Skip the jobs-failed check when the PR carries the bypass-fastfail label.
-            // Lint check above still runs.
-            let labels = [];
-            if (context.payload.pull_request?.labels) {
-              labels = context.payload.pull_request.labels.map(l => l.name);
-            } else {
-              const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                commit_sha: ref,
-              });
-              if (prs.length > 0) {
-                labels = prs[0].labels.map(l => l.name);
-              }
-            }
-            core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
-            if (labels.includes('bypass-fastfail')) {
-              core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
-              return;
-            }
-
-            const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              run_id: context.runId,
-              per_page: 100,
-            });
-            core.info(`[health-check] Total jobs in run: ${jobs.length}`);
-            const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
-            core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
-
-            // Find jobs that failed from a real error, not from fast-fail cascade
-            const rootCauseFailures = jobs.filter(j => {
-              if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
-              // h20 runners are flaky (dirty GPU state from prior runs); their failures
-              // should not cascade fast-fail to other stages. j.name shape from
-              // listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
-              // + optional " (<matrix>)". Split off the base job key before exact
-              // match so we cover both inline + reusable forms without confusing
-              // 'h20' with the 'h200' prefix.
-              const baseName = j.name.split(/[ /]/)[0];
-              if (baseName === 'base-c-test-8-gpu-h20') {
-                core.info(`[health-check] Filtered out h20 job: ${j.name}`);
-                return false;
-              }
-              // multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
-              if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
-                core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
-                return false;
-              }
-              // If the failing step is the health check, it's a cascade — skip it
-              const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
-              if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
-                core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
-                return false;
-              }
-              return true;
-            });
-            core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
-
-            if (rootCauseFailures.length > 0) {
-              core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
-            } else {
-              core.notice('[health-check] PASS: no root cause failures detected');
-            }
+      - uses: ./.github/actions/check-pr-test-health
 
       - name: Install dependencies
         env:

.github/workflows/_npu-single-node-test-stage.yml

@@ -43,7 +43,7 @@ on:
         default: '{}'
         description: 'JSON run metadata {branch_label, workflow_name, create_time}, recorded once at workflow start'
       skip_pr_test_health_check:
-        description: 'Set to true to skip the PR test health check (fast-fail gate).'
+        description: 'Set to true to skip the PR test health check (fail-fast gate).'
         type: string
         default: 'false'
       is_nightly_pipeline_job:
@@ -101,112 +101,7 @@ jobs:
           # checks out the same commit even if the branch advances mid-run.
           ref: ${{ inputs.ref || github.sha }}
 
-      - name: Check PR test health
-        uses: actions/github-script@v8
-        env:
-          SKIP_PR_TEST_HEALTH_CHECK: ${{ env.SKIP_PR_TEST_HEALTH_CHECK }}
-        with:
-          github-token: ${{ inputs.github-token || github.token }}
-          script: |
-            core.notice(`[health-check] START — event=${context.eventName}, runId=${context.runId}`);
-
-            // Skip when explicitly requested via env var (e.g. release branch cut)
-            if (process.env.SKIP_PR_TEST_HEALTH_CHECK === 'true') {
-              core.notice('[health-check] SKIP: SKIP_PR_TEST_HEALTH_CHECK=true');
-              return;
-            }
-
-            // Skip for scheduled runs — they should collect all failures, not fast-fail
-            if (context.eventName === 'schedule') {
-              core.notice('[health-check] SKIP: scheduled run');
-              return;
-            }
-
-            // Check lint status from the separate Lint workflow (lint.yml).
-            // listJobsForWorkflowRun only sees jobs within the SAME run, so we use
-            // checks.listForRef which queries by commit SHA across ALL workflows.
-            const ref = context.payload.pull_request?.head?.sha || context.sha;
-            core.info(`[health-check] Checking lint for ref=${ref}`);
-            const { data } = await github.rest.checks.listForRef({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              ref: ref,
-              check_name: 'lint',
-            });
-            const lintRun = data.check_runs.find(
-              cr => cr.app?.slug === 'github-actions'
-            );
-            core.info(`[health-check] Lint check: status=${lintRun?.status}, conclusion=${lintRun?.conclusion}`);
-            if (lintRun?.status === 'completed' && lintRun?.conclusion === 'failure') {
-              core.setFailed('Fast-fail: lint check failed');
-              return;
-            }
-
-            // Skip the jobs-failed check when the PR carries the bypass-fastfail label.
-            // Lint check above still runs.
-            let labels = [];
-            if (context.payload.pull_request?.labels) {
-              labels = context.payload.pull_request.labels.map(l => l.name);
-            } else {
-              const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                commit_sha: ref,
-              });
-              if (prs.length > 0) {
-                labels = prs[0].labels.map(l => l.name);
-              }
-            }
-            core.info(`[health-check] PR labels: [${labels.join(', ')}]`);
-            if (labels.includes('bypass-fastfail')) {
-              core.notice('[health-check] SKIP jobs-failed check: bypass-fastfail label present');
-              return;
-            }
-
-            const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              run_id: context.runId,
-              per_page: 100,
-            });
-            core.info(`[health-check] Total jobs in run: ${jobs.length}`);
-            const failedJobs = jobs.filter(j => j.status === 'completed' && j.conclusion === 'failure');
-            core.info(`[health-check] Failed jobs (before filtering): ${failedJobs.map(j => `${j.name}(${j.conclusion})`).join(', ') || 'none'}`);
-
-            // Find jobs that failed from a real error, not from fast-fail cascade
-            const rootCauseFailures = jobs.filter(j => {
-              if (j.status !== 'completed' || j.conclusion !== 'failure') return false;
-              // h20 runners are flaky (dirty GPU state from prior runs); their failures
-              // should not cascade fast-fail to other stages. j.name shape from
-              // listJobsForWorkflowRun: "<job-key>" + optional " / <reusable-job>"
-              // + optional " (<matrix>)". Split off the base job key before exact
-              // match so we cover both inline + reusable forms without confusing
-              // 'h20' with the 'h200' prefix.
-              const baseName = j.name.split(/[ /]/)[0];
-              if (baseName === 'base-c-test-8-gpu-h20') {
-                core.info(`[health-check] Filtered out h20 job: ${j.name}`);
-                return false;
-              }
-              // multimodal-gen NPU tests should not cascade fast-fail to perf/accuracy stages.
-              if (baseName === 'multimodal-gen-test-1-npu-a3' || baseName === 'multimodal-gen-test-2-npu-a3') {
-                core.info(`[health-check] Filtered out multimodal-gen NPU job: ${j.name}`);
-                return false;
-              }
-              // If the failing step is the health check, it's a cascade — skip it
-              const failedStep = (j.steps || []).find(s => s.conclusion === 'failure');
-              if (failedStep && (failedStep.name.includes('check-pr-test-health') || failedStep.name.includes('Check PR test health'))) {
-                core.info(`[health-check] Filtered out cascade failure: ${j.name} (failed step: ${failedStep.name})`);
-                return false;
-              }
-              return true;
-            });
-            core.info(`[health-check] Root cause failures (after filtering): ${rootCauseFailures.map(j => j.name).join(', ') || 'none'}`);
-
-            if (rootCauseFailures.length > 0) {
-              core.setFailed(`Fast-fail: skipping — root cause job(s): ${rootCauseFailures.map(j => j.name).join(', ')}`);
-            } else {
-              core.notice('[health-check] PASS: no root cause failures detected');
-            }
+      - uses: ./.github/actions/check-pr-test-health
 
       - name: Check npu info
         run: |

.github/workflows/_pr-test-check-changes.yml

@@ -136,15 +136,24 @@ jobs:
               - "test/registered/rust/**"
               - ".github/workflows/_pr-test-*.yml"
 
+      - name: Resolve CI labels
+        id: ci-labels
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { resolveCiLabels } = require(
+              `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
+            );
+            const axes = await resolveCiLabels(github, context);
+            core.info(`CI labels: [${axes.labels.join(', ')}]`);
+            core.setOutput('max_concurrency', String(axes.maxConcurrency));
+
       - name: Determine full-parallel mode
         id: parallel-mode
         run: |
           # `full=true` lifts the matrix-fanout throttle so each suite's
-          # max_parallel = size. Conditions:
-          #   1. Scheduled cron run.
-          #   2. run_all_tests run (manual full dispatch / release) -- a full
-          #      run should mirror the cron's parallelism, not just its test set.
-          #   3. pull_request event with the `high priority` label.
+          # max_parallel = size.
           FULL=false
           if [[ "${{ github.event_name }}" == "schedule" ]]; then
             FULL=true
@@ -152,9 +161,9 @@ jobs:
           elif [[ "${{ inputs.run_all_tests }}" == "true" ]]; then
             FULL=true
             echo "run_all_tests -> full parallelism"
-          elif [[ "${{ github.event_name }}" == "pull_request" && "${{ contains(github.event.pull_request.labels.*.name, 'high priority') }}" == "true" ]]; then
+          elif [[ "${{ steps.ci-labels.outputs.max_concurrency }}" == "true" ]]; then
             FULL=true
-            echo "high priority PR -> full parallelism"
+            echo "max-concurrency PR -> full parallelism"
           fi
           echo "full=$FULL" >> "$GITHUB_OUTPUT"
 
@@ -189,10 +198,7 @@ jobs:
         id: partitions
         run: |
           # Emit a single JSON output `partitions` keyed by suite name with
-          # {size, arr, max_parallel} fields per suite. Replaces the prior
-          # full/low max-parallel presets; `--full-parallel` keeps the
-          # `high priority` PR / scheduled cron escape hatch.
-          # See scripts/ci/utils/compute_partitions.py.
+          # {size, arr, max_parallel} fields per suite.
           python3 scripts/ci/utils/compute_partitions.py \
             --full-parallel ${{ steps.parallel-mode.outputs.full }} \
             --partition-model-file /tmp/partition-model.json \

.github/workflows/cancel-unfinished-pr-tests.yml

@@ -8,8 +8,8 @@ on:
         required: true
         type: string
         default: 'pr-test.yml pr-test-extra.yml'
-      include_high_priority:
-        description: 'Also cancel runs from high-priority PRs'
+      include_highest_priority:
+        description: 'Also cancel runs from PRs labelled highest-priority'
         required: false
         type: boolean
         default: false
@@ -36,7 +36,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
           WORKFLOWS: ${{ github.event.inputs.workflows }}
-          INCLUDE_HIGH_PRIORITY: ${{ github.event.inputs.include_high_priority }}
+          INCLUDE_HIGHEST_PRIORITY: ${{ github.event.inputs.include_highest_priority }}
           INCLUDE_RERUN_TEST: ${{ github.event.inputs.include_rerun_test }}
         shell: bash
         run: |
@@ -49,7 +49,7 @@ jobs:
           fi
 
           echo "Targeting ${#WORKFLOW_FILES[@]} workflow(s): ${WORKFLOW_FILES[*]}"
-          echo "include_high_priority=$INCLUDE_HIGH_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST"
+          echo "include_highest_priority=$INCLUDE_HIGHEST_PRIORITY, include_rerun_test=$INCLUDE_RERUN_TEST"
           echo ""
 
           # Decide whether to cancel run_id given a PR-lookup endpoint.
@@ -99,12 +99,12 @@ jobs:
               return
             fi
 
-            if echo "$labels" | grep -Fxq "high priority"; then
-              if [ "$INCLUDE_HIGH_PRIORITY" != "true" ]; then
-                echo "  🛑 Skipping (high priority label)"
+            if echo "$labels" | grep -Fxq "highest-priority"; then
+              if [ "$INCLUDE_HIGHEST_PRIORITY" != "true" ]; then
+                echo "  🛑 Skipping (highest-priority label)"
                 return
               fi
-              echo "  ⚠️  High priority PR, but include_high_priority is enabled"
+              echo "  ⚠️  highest-priority PR, but include_highest_priority is enabled"
             fi
 
             echo "  🚫 Cancelling..."

.github/workflows/close-stale-prs.yml

@@ -38,7 +38,7 @@ jobs:
             const QUOTA_IDLE = 7;
             const STALE_DAYS = 90;
 
-            const KEEP_LABELS = new Set(['high priority', 'keep-open', 'good first issue']);
+            const KEEP_LABELS = new Set(['highest-priority', 'keep-open', 'good first issue']);
             const WIP_MARKER = /^\s*\[?\s*(wip|do[ _-]?not[ _-]?merge|dnm|draft)\s*\]?/i;
 
             // Scheduled runs always act; only manual runs can be dry.

.github/workflows/list-active-pr-runs.yml

@@ -127,23 +127,23 @@ jobs:
           # Get unique PR numbers (exclude NO_PR entries)
           pr_numbers=$(cut -d'|' -f1 < "$pr_data_file" | grep -v '^NO_PR$' | sort -u || true)
 
-          # Separate high priority and normal PRs
-          high_priority_prs=()
+          # Separate highest-priority and normal PRs
+          highest_priority_prs=()
           normal_prs=()
 
           for pr_num in $pr_numbers; do
             labels=$(gh pr view "$pr_num" --repo "$REPO" --json labels \
               | jq -r '.labels[].name' 2>/dev/null || true)
 
-            if echo "$labels" | grep -Fxq "high priority"; then
-              high_priority_prs+=($pr_num)
+            if echo "$labels" | grep -Fxq "highest-priority"; then
+              highest_priority_prs+=($pr_num)
             else
               normal_prs+=($pr_num)
             fi
           done
 
-          # Combine: high priority first, then normal
-          sorted_pr_numbers=("${high_priority_prs[@]}" "${normal_prs[@]}")
+          # Combine: highest-priority first, then normal
+          sorted_pr_numbers=("${highest_priority_prs[@]}" "${normal_prs[@]}")
 
           pr_count=0
           total_running=0
@@ -170,8 +170,8 @@ jobs:
 
             # Add priority indicator
             priority_indicator=""
-            if echo "$pr_labels" | grep -q "high priority"; then
-              priority_indicator="🔴 [HIGH PRIORITY] "
+            if echo "$pr_info" | jq -e '[.labels[].name] | index("highest-priority")' >/dev/null; then
+              priority_indicator="🔴 [HIGHEST PRIORITY] "
             fi
 
             echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

.github/workflows/pr-test-amd-extra.yml

@@ -243,7 +243,14 @@ jobs:
         extra-a-test-1-gpu-large-amd,
         extra-a-test-2-gpu-large-amd,
       ]
-    if: always()
+    # Same `labeled` guard as call-gate: an unrelated label would otherwise finish
+    # green with nothing executed, over the real run's result.
+    if: |
+      always() &&
+      (github.event_name != 'pull_request' ||
+       github.event.action != 'labeled' ||
+       github.event.label.name == 'run-ci' ||
+       github.event.label.name == 'run-ci-extra')
     runs-on: ubuntu-latest
     steps:
       - name: Check all dependent job statuses

.github/workflows/pr-test-amd.yml

@@ -187,19 +187,29 @@ jobs:
             echo "Run mode: FILTERED (triggered by ${{ github.event_name }})"
           fi
 
+      - name: Resolve CI labels
+        id: ci-labels
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { resolveCiLabels } = require(
+              `${process.env.GITHUB_WORKSPACE}/.github/scripts/ci-labels.cjs`
+            );
+            const axes = await resolveCiLabels(github, context);
+            core.info(`CI labels: [${axes.labels.join(', ')}]`);
+            core.setOutput('bypass_fail_fast', String(axes.bypassFailFast));
+
       - name: Set continue-on-error for schedule/full runs
         id: set-continue-on-error
         env:
-          # `bypass-fastfail` PR label: also disable within-suite fast-fail
-          # here. The shared actions/wait-for-jobs already honors the same
-          # label to skip cross-stage waits.
-          BYPASS_FASTFAIL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'bypass-fastfail') }}
+          BYPASS_FAIL_FAST: ${{ steps.ci-labels.outputs.bypass_fail_fast }}
         run: |
           if [[ "${{ steps.run-mode.outputs.run_all_tests }}" == "true" \
              || "${{ inputs.continue_on_error }}" == "true" \
-             || "$BYPASS_FASTFAIL_LABEL" == "true" ]]; then
+             || "$BYPASS_FAIL_FAST" == "true" ]]; then
             echo "continue_on_error=true" >> $GITHUB_OUTPUT
-            echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fastfail=$BYPASS_FASTFAIL_LABEL)"
+            echo "Continue-on-error: ENABLED (run_all_tests=${{ steps.run-mode.outputs.run_all_tests }}, input=${{ inputs.continue_on_error }}, bypass-fail-fast=$BYPASS_FAIL_FAST)"
           else
             echo "continue_on_error=false" >> $GITHUB_OUTPUT
             echo "Continue-on-error: DISABLED"

.github/workflows/pr-test-extra.yml

@@ -50,13 +50,14 @@ on:
         type: boolean
         default: false
       skip_pr_test_health_check:
-        description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
+        description: "Skip PR test health check fail-fast (e.g. for release branch cuts)"
         required: false
         type: boolean
         default: false
 
 concurrency:
-  group: pr-test-extra-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
+  # Keys on the PR number: two forks can share a head_ref and would cancel each other.
+  group: pr-test-extra-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
   cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
 
 env:
@@ -202,7 +203,14 @@ jobs:
         simulator-test-cpu,
         extra-test,
       ]
-    if: always()
+    # Without this guard an unrelated label starts a run that finishes green with
+    # nothing executed, over the real failure; `skipped` is what pr-states.yml reads.
+    if: |
+      always() &&
+      (github.event_name != 'pull_request' ||
+       github.event.action != 'labeled' ||
+       github.event.label.name == 'run-ci' ||
+       github.event.label.name == 'run-ci-extra')
     runs-on: ubuntu-latest
     steps:
       - name: Check all dependent job statuses
@@ -232,10 +240,14 @@ jobs:
   # there (initial completion) and pull_request_target (push / label).
   notify-pr-states:
     needs: [pr-test-extra-finish]
+    # Same guard; pr-states.yml subscribes to labeled/unlabeled itself, so nothing is lost.
     if: |
       always() &&
       github.event_name == 'pull_request' &&
-      github.event.pull_request.head.repo.full_name == github.repository
+      github.event.pull_request.head.repo.full_name == github.repository &&
+      (github.event.action != 'labeled' ||
+       github.event.label.name == 'run-ci' ||
+       github.event.label.name == 'run-ci-extra')
     runs-on: ubuntu-latest
     steps:
       - name: Dispatch pr-states refresh

.github/workflows/pr-test.yml

@@ -41,17 +41,17 @@ on:
         type: boolean
         default: false
       skip_pr_test_health_check:
-        description: "Skip PR test health check fast-fail (e.g. for release branch cuts)"
+        description: "Skip PR test health check fail-fast (e.g. for release branch cuts)"
         required: false
         type: boolean
         default: false
 
 concurrency:
-  # Concurrency group structure: pr-test-{event}-{branch}-{git_ref}
   # - event_name prevents scheduled runs from colliding with fork PRs whose branch is named 'main'
   #   (without it, both resolve the branch segment to 'main' and block each other)
-  # - github.head_ref (pull_request) or github.ref_name (workflow_dispatch) normalizes to branch name
-  group: pr-test-${{ github.event_name }}-${{ github.head_ref || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
+  # - a PR keys on its number: github.head_ref is a bare branch name with no owner,
+  #   so two forks using the same name would share one group and cancel each other
+  group: pr-test-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || 'default' }}-${{ inputs.git_ref || 'all' }}
   cancel-in-progress: ${{ github.event_name != 'workflow_call' }}
 
 env:
@@ -88,11 +88,6 @@ jobs:
     secrets: inherit
 
   # =============================================== Wait Jobs for Sequential PR Execution ====================================================
-  # These jobs poll GitHub API to wait for previous stages to complete.
-  # For PR runs: wait jobs run and enforce sequential execution via polling.
-  # For scheduled runs: wait jobs are skipped, enabling parallel execution for easier retry.
-  # For PRs with the `bypass-fastfail` label: wait jobs run but return success immediately
-  # (handled inside the wait-for-jobs action), so downstream stages dispatch in parallel.
 
   wait-for-base-a:
     needs: [check-changes, call-gate]