[CI] Move tests onto the right CI stages (#34074)

Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>

.github/workflows/_pr-test-stage.yml

@@ -101,9 +101,17 @@ jobs:
     name: ${{ inputs.self_name }} (${{ matrix.partition }})
     # Runs on schedule / parallel-dispatch / non-failed PR with main_package or sgl_kernel changes.
     # Temporarily skip the broken GB300 runner before GitHub tries to allocate it.
+    # `runner_filter` is matched here rather than in each caller's job `if`: a
+    # caller that declares its runners as a matrix cannot, since job-level `if`
+    # sees github/needs/vars/inputs but not `matrix`. Callers that never declare
+    # the input read back null and match every runner_config.
     if: |
       always() &&
       inputs.runner_config != '4-gpu-gb300' &&
+      (fromJson(inputs.caller_inputs).runner_filter == null ||
+       fromJson(inputs.caller_inputs).runner_filter == '' ||
+       fromJson(inputs.caller_inputs).runner_filter == 'all' ||
+       fromJson(inputs.caller_inputs).runner_filter == inputs.runner_config) &&
       ((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
       (fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true')
     # runs-on resolved from runs_on_map; check-changes already substituted

.github/workflows/weekly-test-cpu.yml

@@ -0,0 +1,49 @@
+# Weekly CPU tests. Holds the CPU-only unit tests that are not worth a
+# per-commit slot -- debug tooling (dump comparator, source patcher) whose
+# breakage is fine to notice once a week.
+#
+# Separate from weekly-test-nvidia.yml because CPU goes through a different
+# reusable workflow, and `uses:` takes no expressions -- so it cannot be another
+# row in that file's matrix. Both run the same shared stage their per-commit
+# counterparts do, so a test behaves identically in a PR and in the weekly run.
+name: Weekly Test (CPU)
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Sunday 00:00 UTC, alongside the Nvidia weekly run
+  workflow_dispatch:
+
+concurrency:
+  group: weekly-test-cpu-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+  pull-requests: read
+
+jobs:
+  # run_all_tests skips the paths-filter, so every test runs regardless of what
+  # the last commit touched.
+  check-changes:
+    uses: ./.github/workflows/_pr-test-check-changes.yml
+    with:
+      pr_test_yml: '.github/workflows/weekly-test-cpu.yml'
+      run_all_tests: true
+      force_continue_on_error: true
+    secrets: inherit
+
+  # No rust_ext_artifact: nothing builds one here, so the stage falls back to
+  # its cache and compiles on a miss.
+  weekly-test-cpu:
+    needs: check-changes
+    if: github.repository == 'sgl-project/sglang'
+    uses: ./.github/workflows/_pr-test-stage-cpu.yml
+    with:
+      self_name: weekly-test-cpu
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '90'
+    secrets: inherit

.github/workflows/weekly-test-nvidia.yml

@@ -8,13 +8,13 @@ on:
   workflow_dispatch:
     inputs:
       runner_filter:
-        description: 'Select which runner_config to run (leave empty or "all" to run all)'
+        # A free-text field rather than a choice: `on:` blocks take no
+        # expressions, so an options list would be a second copy of the matrix
+        # below to keep in sync. Matched in _pr-test-stage.yml.
+        description: 'runner_config to run alone (empty or "all" runs every one)'
         required: false
-        type: choice
+        type: string
         default: 'all'
-        options:
-          - 'all'
-          - '8-gpu-h200'
       full_parallel:
         description: 'Run all shards of a job at once (faster, but competes with per-commit CI for machines). Off by default: one shard at a time.'
         required: false
@@ -43,17 +43,33 @@ jobs:
       force_continue_on_error: true
     secrets: inherit
 
-  weekly-test-8-gpu-h200:
+  # One row per machine type; timeouts mirror the same runner's nightly job.
+  # Adding a machine is one row here, and adding a test to an existing machine
+  # is none -- `stage="weekly"` + `runner_config=` on the test is enough.
+  weekly-test:
+    # Without this, the matrix default names the job after every value in the
+    # row -- "weekly-test (1-gpu-large, 120, 180)".
+    name: weekly-test-${{ matrix.runner_config }}
     needs: check-changes
-    if: github.repository == 'sgl-project/sglang' && (inputs.runner_filter == '' || inputs.runner_filter == 'all' || inputs.runner_filter == '8-gpu-h200')
+    if: github.repository == 'sgl-project/sglang'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { runner_config: 1-gpu-large, run_timeout: '120', job_timeout: '180' }
+          - { runner_config: 2-gpu-large, run_timeout: '240', job_timeout: '300' }
+          - { runner_config: 4-gpu-h100, run_timeout: '120', job_timeout: '180' }
+          - { runner_config: 4-gpu-b200, run_timeout: '150', job_timeout: '210' }
+          - { runner_config: 8-gpu-h200, run_timeout: '240', job_timeout: '300' }
+          - { runner_config: 8-gpu-b200, run_timeout: '360', job_timeout: '420' }
     uses: ./.github/workflows/_pr-test-stage.yml
     with:
-      self_name: weekly-test-8-gpu-h200
-      runner_config: 8-gpu-h200
+      self_name: weekly-test-${{ matrix.runner_config }}
+      runner_config: ${{ matrix.runner_config }}
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
-      run_timeout_minutes: '240'
-      job_timeout_minutes: '300'
+      run_timeout_minutes: ${{ matrix.run_timeout }}
+      job_timeout_minutes: ${{ matrix.job_timeout }}
       scheduled: true
     secrets: inherit