[CI] Move tests onto the right CI stages (#34074)

Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>

.github/workflows/_pr-test-stage.yml

@@ -101,9 +101,17 @@ jobs:
     name: ${{ inputs.self_name }} (${{ matrix.partition }})
     # Runs on schedule / parallel-dispatch / non-failed PR with main_package or sgl_kernel changes.
     # Temporarily skip the broken GB300 runner before GitHub tries to allocate it.
+    # `runner_filter` is matched here rather than in each caller's job `if`: a
+    # caller that declares its runners as a matrix cannot, since job-level `if`
+    # sees github/needs/vars/inputs but not `matrix`. Callers that never declare
+    # the input read back null and match every runner_config.
     if: |
       always() &&
       inputs.runner_config != '4-gpu-gb300' &&
+      (fromJson(inputs.caller_inputs).runner_filter == null ||
+       fromJson(inputs.caller_inputs).runner_filter == '' ||
+       fromJson(inputs.caller_inputs).runner_filter == 'all' ||
+       fromJson(inputs.caller_inputs).runner_filter == inputs.runner_config) &&
       ((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) &&
       (fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true')
     # runs-on resolved from runs_on_map; check-changes already substituted

.github/workflows/weekly-test-cpu.yml

@@ -0,0 +1,49 @@
+# Weekly CPU tests. Holds the CPU-only unit tests that are not worth a
+# per-commit slot -- debug tooling (dump comparator, source patcher) whose
+# breakage is fine to notice once a week.
+#
+# Separate from weekly-test-nvidia.yml because CPU goes through a different
+# reusable workflow, and `uses:` takes no expressions -- so it cannot be another
+# row in that file's matrix. Both run the same shared stage their per-commit
+# counterparts do, so a test behaves identically in a PR and in the weekly run.
+name: Weekly Test (CPU)
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'  # Sunday 00:00 UTC, alongside the Nvidia weekly run
+  workflow_dispatch:
+
+concurrency:
+  group: weekly-test-cpu-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  actions: write
+  contents: read
+  issues: read
+  pull-requests: read
+
+jobs:
+  # run_all_tests skips the paths-filter, so every test runs regardless of what
+  # the last commit touched.
+  check-changes:
+    uses: ./.github/workflows/_pr-test-check-changes.yml
+    with:
+      pr_test_yml: '.github/workflows/weekly-test-cpu.yml'
+      run_all_tests: true
+      force_continue_on_error: true
+    secrets: inherit
+
+  # No rust_ext_artifact: nothing builds one here, so the stage falls back to
+  # its cache and compiles on a miss.
+  weekly-test-cpu:
+    needs: check-changes
+    if: github.repository == 'sgl-project/sglang'
+    uses: ./.github/workflows/_pr-test-stage-cpu.yml
+    with:
+      self_name: weekly-test-cpu
+      check_changes: ${{ toJson(needs.check-changes.outputs) }}
+      caller_inputs: ${{ toJson(inputs) }}
+      partitions: ${{ needs.check-changes.outputs.partitions }}
+      run_timeout_minutes: '90'
+    secrets: inherit

.github/workflows/weekly-test-nvidia.yml

@@ -8,13 +8,13 @@ on:
   workflow_dispatch:
     inputs:
       runner_filter:
-        description: 'Select which runner_config to run (leave empty or "all" to run all)'
+        # A free-text field rather than a choice: `on:` blocks take no
+        # expressions, so an options list would be a second copy of the matrix
+        # below to keep in sync. Matched in _pr-test-stage.yml.
+        description: 'runner_config to run alone (empty or "all" runs every one)'
         required: false
-        type: choice
+        type: string
         default: 'all'
-        options:
-          - 'all'
-          - '8-gpu-h200'
       full_parallel:
         description: 'Run all shards of a job at once (faster, but competes with per-commit CI for machines). Off by default: one shard at a time.'
         required: false
@@ -43,17 +43,33 @@ jobs:
       force_continue_on_error: true
     secrets: inherit
 
-  weekly-test-8-gpu-h200:
+  # One row per machine type; timeouts mirror the same runner's nightly job.
+  # Adding a machine is one row here, and adding a test to an existing machine
+  # is none -- `stage="weekly"` + `runner_config=` on the test is enough.
+  weekly-test:
+    # Without this, the matrix default names the job after every value in the
+    # row -- "weekly-test (1-gpu-large, 120, 180)".
+    name: weekly-test-${{ matrix.runner_config }}
     needs: check-changes
-    if: github.repository == 'sgl-project/sglang' && (inputs.runner_filter == '' || inputs.runner_filter == 'all' || inputs.runner_filter == '8-gpu-h200')
+    if: github.repository == 'sgl-project/sglang'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { runner_config: 1-gpu-large, run_timeout: '120', job_timeout: '180' }
+          - { runner_config: 2-gpu-large, run_timeout: '240', job_timeout: '300' }
+          - { runner_config: 4-gpu-h100, run_timeout: '120', job_timeout: '180' }
+          - { runner_config: 4-gpu-b200, run_timeout: '150', job_timeout: '210' }
+          - { runner_config: 8-gpu-h200, run_timeout: '240', job_timeout: '300' }
+          - { runner_config: 8-gpu-b200, run_timeout: '360', job_timeout: '420' }
     uses: ./.github/workflows/_pr-test-stage.yml
     with:
-      self_name: weekly-test-8-gpu-h200
+      self_name: weekly-test-${{ matrix.runner_config }}
-      runner_config: 8-gpu-h200
+      runner_config: ${{ matrix.runner_config }}
       check_changes: ${{ toJson(needs.check-changes.outputs) }}
       caller_inputs: ${{ toJson(inputs) }}
       partitions: ${{ needs.check-changes.outputs.partitions }}
-      run_timeout_minutes: '240'
+      run_timeout_minutes: ${{ matrix.run_timeout }}
-      job_timeout_minutes: '300'
+      job_timeout_minutes: ${{ matrix.job_timeout }}
       scheduled: true
     secrets: inherit

python/sglang/test/test_utils.py

@@ -147,12 +147,13 @@ DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8"
 DEFAULT_ENABLE_ROUTED_EXPERTS_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B"
 
 # Nightly tests
-DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = (
+# Deliberate omission: a model another registered suite already uses as its base
-    "meta-llama/Llama-3.1-8B-Instruct,Qwen/Qwen3-8B,Qwen/Qwen3-4B"
+# model is left out, since a regression there surfaces in that suite instead.
+DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = (
+    "meta-llama/Llama-3.1-70B-Instruct,Qwen/Qwen2-57B-A14B-Instruct"
 )
-DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct"
+DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8"
-DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8"
+DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8"
-DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8"
 DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1 = "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4,hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4,hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4"
 DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN = "Qwen/Qwen2.5-1.5B-Instruct"
 DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST = "Qwen/Qwen2.5-VL-3B-Instruct"

scripts/ci/utils/compute_partitions.py

@@ -11,6 +11,7 @@ import importlib.util
 import json
 import math
 import os
+import re
 from collections import defaultdict
 from datetime import datetime, timedelta
 
@@ -49,6 +50,18 @@ _REUSABLE_STAGE_USES = {
 }
 
 
+_MATRIX_REF = re.compile(r"\$\{\{\s*matrix\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}")
+
+
+def _resolve_matrix_refs(value, row: dict):
+    """Substitute `${{ matrix.key }}` from one `strategy.matrix.include` row.
+
+    A caller may declare its runners as a matrix instead of one job each, which
+    leaves `self_name` and `run_timeout_minutes` as expressions here -- GitHub
+    resolves them at dispatch, this static read has to do it itself."""
+    return _MATRIX_REF.sub(lambda m: str(row[m.group(1)]), str(value))
+
+
 def load_run_timeouts(pr_test_yml_path: str) -> dict:
     """Map `self_name -> run_timeout_minutes` from one pr-test*.yml. The input
     is required in both reusable stage workflows -- KeyError surfaces missing."""
@@ -60,8 +73,17 @@ def load_run_timeouts(pr_test_yml_path: str) -> dict:
         if not isinstance(job, dict) or job.get("uses") not in _REUSABLE_STAGE_USES:
             continue
         with_ = job.get("with") or {}
-        suite = with_.get("self_name", job_id)
+        rows = ((job.get("strategy") or {}).get("matrix") or {}).get("include") or [
-        timeouts[suite] = int(with_["run_timeout_minutes"])
+            None
+        ]
+        for row in rows:
+            if row is None:
+                suite = with_.get("self_name", job_id)
+                timeout = with_["run_timeout_minutes"]
+            else:
+                suite = _resolve_matrix_refs(with_.get("self_name", job_id), row)
+                timeout = _resolve_matrix_refs(with_["run_timeout_minutes"], row)
+            timeouts[suite] = int(timeout)
     if not timeouts:
         raise RuntimeError(
             f"load_run_timeouts: no jobs matched uses in {_REUSABLE_STAGE_USES!r} "

test/registered/8-gpu-models/test_glm_46.py

@@ -7,8 +7,8 @@ from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
 # Runs on both H200 and B200: registered once per runner_config below
-register_cuda_ci(est_time=1320, stage="nightly", runner_config="8-gpu-h200")
+register_cuda_ci(est_time=1320, stage="weekly", runner_config="8-gpu-h200")
-register_cuda_ci(est_time=1320, stage="nightly", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=1320, stage="weekly", runner_config="8-gpu-b200")
 
 GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
 

test/registered/8-gpu-models/test_minimax_m25.py

@@ -7,8 +7,8 @@ from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
 # Runs on both H200 and B200: registered once per runner_config below
-register_cuda_ci(est_time=1860, stage="nightly", runner_config="8-gpu-h200")
+register_cuda_ci(est_time=1860, stage="weekly", runner_config="8-gpu-h200")
-register_cuda_ci(est_time=1860, stage="nightly", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=1860, stage="weekly", runner_config="8-gpu-b200")
 
 MINIMAX_M25_MODEL_PATH = "MiniMaxAI/MiniMax-M2.5"
 

test/registered/8-gpu-models/test_mistral_large3.py

@@ -10,8 +10,8 @@ from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
 
 # Runs on both H200 and B200: registered once per runner_config below
 # Note: trtllm_mla backend may have hardware-specific behavior
-register_cuda_ci(est_time=3000, stage="nightly", runner_config="8-gpu-h200")
+register_cuda_ci(est_time=3000, stage="weekly", runner_config="8-gpu-h200")
-register_cuda_ci(est_time=3000, stage="nightly", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=3000, stage="weekly", runner_config="8-gpu-b200")
 
 MISTRAL_LARGE3_FP8_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
 MISTRAL_LARGE3_NVFP4_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4"

test/registered/8-gpu-models/test_ring_2_5_1t.py

@@ -5,8 +5,8 @@ from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.run_combined_tests import run_combined_tests
 from sglang.test.test_utils import ModelLaunchSettings
 
-register_cuda_ci(est_time=1680, stage="nightly", runner_config="8-gpu-h200")
+register_cuda_ci(est_time=1680, stage="weekly", runner_config="8-gpu-h200")
-register_cuda_ci(est_time=1680, stage="nightly", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=1680, stage="weekly", runner_config="8-gpu-b200")
 
 RING_2_5_1T_MODEL_PATH = "inclusionAI/Ring-2.5-1T"
 

test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py

@@ -20,7 +20,6 @@ from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
-    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -80,9 +79,6 @@ def remove_failing_models(model_str):
     return ",".join(filtered)
 
 
-DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models(
-    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
-)
 DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models(
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2
 )
@@ -206,7 +202,6 @@ class TestNightlyGsm8KEval(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.model_groups = [
-            (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
             (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
             (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
             (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),

test/registered/attention/test_deepseek_v3_deterministic.py

@@ -12,7 +12,7 @@ from sglang.test.test_deterministic_utils import (
     TestDeterministicBase,
 )
 
-register_cuda_ci(est_time=240, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=240, stage="weekly", runner_config="1-gpu-large")
 
 DEEPSEEK_MODEL = "lmsys/sglang-ci-dsv3-test"
 

test/registered/attention/test_qwen3_next_deterministic.py

@@ -12,7 +12,7 @@ from sglang.test.test_deterministic_utils import (
     TestDeterministicBase,
 )
 
-register_cuda_ci(est_time=200, stage="nightly", runner_config="4-gpu-h100")
+register_cuda_ci(est_time=200, stage="weekly", runner_config="4-gpu-h100")
 
 QWEN3_NEXT = "Qwen/Qwen3-Next-80B-A3B-Instruct"
 

test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py

@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
     write_github_step_summary,
 )
 
-register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-b200")
+register_cuda_ci(est_time=900, stage="extra-b", runner_config="4-gpu-b200")
 
 FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
 SERVER_LAUNCH_TIMEOUT = 1000

test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py

@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_cuda_ci(est_time=1770, stage="nightly", runner_config="4-gpu-b200")
+register_cuda_ci(est_time=1770, stage="weekly", runner_config="4-gpu-b200")
 
 
 class FlashinferTrtllmGenMoeBackendFP8Base:

test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py

@@ -11,7 +11,7 @@ import unittest
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase
 
-register_cuda_ci(est_time=130, stage="nightly", runner_config="4-gpu-h100")
+register_cuda_ci(est_time=130, stage="weekly", runner_config="4-gpu-h100")
 
 
 class TestPCGWithEAGLE3(PCGSpecBase, unittest.TestCase):

test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_dflash.py

@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
     CustomTestCase,
 )
 
-register_cuda_ci(est_time=110, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=110, stage="weekly", runner_config="1-gpu-large")
 
 
 class TestPCGWithDFlash(PCGSpecBase, CustomTestCase):

test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py

@@ -8,7 +8,7 @@ import unittest
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase
 
-register_cuda_ci(est_time=450, stage="nightly", runner_config="4-gpu-h100")
+register_cuda_ci(est_time=450, stage="weekly", runner_config="4-gpu-h100")
 
 
 class TestPCGWithMTP(PCGSpecBase, unittest.TestCase):

test/registered/debug_utils/comparator/aligner/entrypoint/test_executor.py

@@ -33,8 +33,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 class TestExecuteSubPlans:

test/registered/debug_utils/comparator/aligner/entrypoint/test_planner.py

@@ -25,8 +25,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 def _make_meta(

test/registered/debug_utils/comparator/aligner/reorderer/test_executor.py

@@ -26,8 +26,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 
 def _zigzag_order(cp_size: int) -> list[int]:

test/registered/debug_utils/comparator/aligner/reorderer/test_planner.py

@@ -26,8 +26,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 class TestComputeReordererPlans:

test/registered/debug_utils/comparator/aligner/test_axis_aligner.py

@@ -17,8 +17,7 @@ from sglang.srt.debug_utils.comparator.log_sink import log_sink
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 class TestComputeAxisAlignerPlan:

test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_loader.py

@@ -19,7 +19,7 @@ from sglang.srt.debug_utils.comparator.log_sink import LogSink
 from sglang.srt.debug_utils.comparator.output_types import ErrorLog, InfoLog
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
 
 _sglang_plugin = _SGLangPlugin()
 _megatron_plugin = _MegatronPlugin()

test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_plugins.py

@@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.aligner.token_aligner.smart.types import
 from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 _sglang_plugin = _SGLangPlugin()
 _megatron_plugin = _MegatronPlugin()

test/registered/debug_utils/comparator/aligner/token_aligner/test_concat_steps.py

@@ -10,8 +10,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import apply_dim_names
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 
 class TestExecuteConcat:

test/registered/debug_utils/comparator/aligner/token_aligner/test_executor.py

@@ -29,8 +29,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 def _named(tensor: torch.Tensor, names: list[str]) -> torch.Tensor:

test/registered/debug_utils/comparator/aligner/token_aligner/test_planner.py

@@ -23,8 +23,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 
 class TestBuildTokenIndexSGLangThd:

test/registered/debug_utils/comparator/aligner/token_aligner/test_thd_seq_lens_loader.py

@@ -14,7 +14,7 @@ from sglang.srt.debug_utils.comparator.aligner.token_aligner.smart.aux_plugins i
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
 
 
 def _save_pt(

test/registered/debug_utils/comparator/aligner/unsharder/test_executor.py

@@ -30,8 +30,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 from sglang.srt.debug_utils.comparator.output_types import ReplicatedCheckResult
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 def _name_tensors(

test/registered/debug_utils/comparator/aligner/unsharder/test_parallel_info.py

@@ -9,8 +9,7 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import AxisInfo
 from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 
 class TestNormalizeParallelInfo:

test/registered/debug_utils/comparator/aligner/unsharder/test_planner.py

@@ -19,8 +19,7 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import (
 from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis, parse_dims
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 class TestComputeUnsharderPlan:

test/registered/debug_utils/comparator/dims_spec/test_dim_parser.py

@@ -12,8 +12,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=6, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 
 class TestParseDim:

test/registered/debug_utils/comparator/dims_spec/test_dims_parser.py

@@ -15,8 +15,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 class TestSingletonDimUtilFilterOut:

test/registered/debug_utils/comparator/dims_spec/test_tensor_naming.py

@@ -14,8 +14,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=7, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 class TestFindDimIndex:

test/registered/debug_utils/comparator/dims_spec/test_types.py

@@ -9,8 +9,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 class TestDimConstants:

test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py

@@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import DiffInfo
 from sglang.srt.debug_utils.comparator.threshold_dsl import DiffThresholdRule
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=20, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=20, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 class TestComputeTensorInfo:

test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py

@@ -63,7 +63,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
 
 _DEFAULT_PERCENTILE_LINES: list[str] = [
     "      [blue]p1        [/]    -1.8000      -1.8000   [dim]+0.00e+00[/]",

test/registered/debug_utils/comparator/tensor_comparator/test_types.py

@@ -21,8 +21,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 def _make_stats(**overrides) -> TensorStats:

test/registered/debug_utils/comparator/test_bundle_comparator.py

@@ -15,7 +15,7 @@ from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.srt.debug_utils.dump_loader import ValueWithMeta
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
 
 
 def _save_tensor(

test/registered/debug_utils/comparator/test_bundle_matcher.py

@@ -13,7 +13,7 @@ from sglang.srt.debug_utils.comparator.bundle_matcher import (
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
 
 
 def _make_row(

test/registered/debug_utils/comparator/test_display.py

@@ -21,7 +21,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
 
 
 def _render_rich(renderable: object) -> str:

test/registered/debug_utils/comparator/test_dp_utils.py

@@ -12,7 +12,7 @@ from sglang.srt.debug_utils.comparator.dp_utils import (
 from sglang.srt.debug_utils.dump_loader import ValueWithMeta
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
 
 
 def _make_sglang_meta(

test/registered/debug_utils/comparator/test_dump_loader.py

@@ -7,7 +7,7 @@ import torch
 from sglang.srt.debug_utils.dump_loader import read_tokenizer_path
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
 
 
 def _save_pt(

test/registered/debug_utils/comparator/test_e2e_demo.py

@@ -26,7 +26,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
 from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="default", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
 
 _EXP_NAME = "demo_exp"
 

test/registered/debug_utils/comparator/test_entrypoint.py

@@ -30,7 +30,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
 from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper, _RecomputeStatus
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
 
 _FIXED_EXP_NAME = "my_exp_name"
 

test/registered/debug_utils/comparator/test_log_sink.py

@@ -11,8 +11,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
 from sglang.srt.debug_utils.comparator.report_sink import report_sink
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 def _make_error_log(**overrides) -> ErrorLog:

test/registered/debug_utils/comparator/test_manually_verify.py

@@ -23,7 +23,7 @@ import torch
 
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=60, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=60, stage="weekly", runner_config="cpu")
 
 _PUBLISH_DIR: Path = Path("/tmp/comparator_manual_verify")
 _PNG_MAGIC: bytes = b"\x89PNG"

test/registered/debug_utils/comparator/test_meta_overrider.py

@@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.meta_overrider import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 # ───────────────────── Unit: MetaOverrideRule ─────────────────────

test/registered/debug_utils/comparator/test_model_validation.py

@@ -44,8 +44,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
 from sglang.srt.debug_utils.comparator.utils import Pair, _check_equal_lengths
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 class TestCheckEqualLengths:

test/registered/debug_utils/comparator/test_output_types.py

@@ -60,7 +60,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
 from sglang.srt.debug_utils.comparator.utils import Pair
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
 
 
 def _render_rich(renderable: object) -> str:

test/registered/debug_utils/comparator/test_per_token_visualizer.py

@@ -15,7 +15,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.comparator import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
 
 _PNG_MAGIC: bytes = b"\x89PNG"
 

test/registered/debug_utils/comparator/test_preset.py

@@ -3,8 +3,7 @@ import pytest
 from sglang.srt.debug_utils.comparator.preset import PRESETS, expand_preset
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=8, suite="base-c-test-cpu")
 
 
 class TestExpandPreset:

test/registered/debug_utils/comparator/test_threshold_dsl.py

@@ -11,7 +11,7 @@ from sglang.srt.debug_utils.comparator.threshold_dsl import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
 
 
 def _ev(

test/registered/debug_utils/comparator/test_utils.py

@@ -17,8 +17,7 @@ from sglang.srt.debug_utils.comparator.utils import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=7, suite="base-c-test-cpu")
 
 
 class TestCalcRelDiff:

test/registered/debug_utils/comparator/test_visualizer.py

@@ -10,7 +10,7 @@ from sglang.srt.debug_utils.comparator.visualizer.preprocessing import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
 
 
 class TestPreprocessTensor:

test/registered/debug_utils/comparator/testing_helpers.py

@@ -12,8 +12,8 @@ from sglang.test.ci.ci_register import register_cpu_ci
 
 register_cpu_ci(
     est_time=0,
-    suite="base-a-test-cpu",
+    stage="weekly",
-    nightly=True,
+    runner_config="cpu",
     disabled="helper module, no tests",
 )
 

test/registered/debug_utils/source_patcher/test_code_patcher.py

@@ -10,8 +10,7 @@ from sglang.srt.debug_utils.source_patcher.code_patcher import (
 from sglang.srt.debug_utils.source_patcher.types import EditSpec, PatchSpec
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=5, suite="base-c-test-cpu")
 
 SAMPLE_MODULE_NAME = "_source_patcher_test_fixtures.sample_module"
 

test/registered/debug_utils/source_patcher/test_dumper_integration.py

@@ -8,8 +8,7 @@ import yaml
 from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=6, suite="base-c-test-cpu")
 
 SAMPLE_MODULE_NAME = "_source_patcher_test_fixtures.sample_module"
 

test/registered/debug_utils/source_patcher/test_source_editor.py

@@ -5,8 +5,7 @@ from sglang.srt.debug_utils.source_patcher.source_editor import apply_edits
 from sglang.srt.debug_utils.source_patcher.types import EditSpec, PatchApplicationError
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=4, suite="base-c-test-cpu")
 
 
 class TestApplyEdits:

test/registered/debug_utils/test_dump_comparator.py

@@ -9,7 +9,7 @@ from sglang.srt.debug_utils.dump_comparator import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
 
 
 # ----------------------------- Unit tests -----------------------------

test/registered/debug_utils/test_dump_loader.py

@@ -15,7 +15,7 @@ from sglang.srt.debug_utils.dump_loader import (
 )
 from sglang.test.ci.ci_register import register_cpu_ci
 
-register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
 
 
 class TestReadMeta:

test/registered/debug_utils/test_schedule_simulator.py

@@ -25,8 +25,7 @@ from sglang.srt.debug_utils.schedule_simulator import (
 from sglang.test.ci.ci_register import register_cpu_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cpu_ci(est_time=120, suite="base-a-test-cpu", nightly=True)
+register_cpu_ci(est_time=357, stage="weekly", runner_config="cpu")
-register_cpu_ci(est_time=357, suite="base-c-test-cpu")
 
 # ==================== Non-E2E Tests ====================
 

test/registered/eval/test_text_models_gsm8k_eval.py

@@ -9,7 +9,6 @@ from sglang.test.run_eval import run_eval
 from sglang.test.test_utils import (
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
-    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
     DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
     DEFAULT_URL_FOR_TEST,
     ModelLaunchSettings,
@@ -23,21 +22,15 @@ from sglang.test.test_utils import (
 # downloading on cache miss. Use a longer timeout than the default 600s.
 NIGHTLY_EVAL_SERVER_TIMEOUT = 1800
 
-register_cuda_ci(est_time=2880, stage="nightly", runner_config="2-gpu-large")
+register_cuda_ci(est_time=2880, stage="weekly", runner_config="2-gpu-large")
 
 MODEL_SCORE_THRESHOLDS = {
     # sgl-eval (zero-shot chat, \boxed{}, math_verify grading). Thresholds are
     # measured_score - 0.05, baselined on H100 2-GPU over the full 1319 split.
-    "meta-llama/Llama-3.1-8B-Instruct": 0.77,  # 81.05% measured - 5%
-    "Qwen/Qwen3-8B": 0.76,  # 81.43% measured - 5%
-    "Qwen/Qwen3-4B": 0.77,  # 82.41% measured - 5%
     "meta-llama/Llama-3.1-70B-Instruct": 0.90,  # 94.77% measured - 5%
-    "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.39,  # 43.52% measured - 5%
     "Qwen/Qwen2-57B-A14B-Instruct": 0.46,  # 50.87% measured - 5%
-    "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.77,  # 82.34% measured - 5%
     "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.23,  # 27.82% measured - 5%
     "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.80,  # 84.91% measured - 5%
-    "zai-org/GLM-4.5-Air-FP8": 0.73,  # 77.48% measured - 5%
     "neuralmagic/gemma-2-2b-it-FP8": 0.02,  # 6.52% measured - 5%
     "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89,  # 94.01% measured - 5%
     "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.35,  # 40.33% measured - 5%
@@ -51,9 +44,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.models = []
-        models_tp1 = parse_models(
+        models_tp1 = parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1)
-            DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
-        ) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1)
         for model_path in models_tp1:
             cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
 

test/registered/kernels/benchmark/kv_canary/bench_plan.py

@@ -25,7 +25,7 @@ from sglang.kernels.ops.kv_canary.verify import VerifyPlan
 from sglang.kernels.ops.kv_canary.write import WritePlan
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
-register_cuda_ci(est_time=40, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=40, stage="weekly", runner_config="1-gpu-large")
 # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
 register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True)
 

test/registered/kernels/benchmark/kv_canary/bench_scatter_req_token_ids.py

@@ -16,7 +16,7 @@ from sglang.kernels.ops.kv_canary.scatter_req_token_ids import (
 )
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
-register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large")
 # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
 # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
 # (_BS_AXIS_CI/_SEQ_LEN_AXIS_CI via get_benchmark_range), same as CUDA nightly.

test/registered/kernels/benchmark/kv_canary/bench_verify.py

@@ -32,7 +32,7 @@ from sglang.kernels.ops.kv_canary.verify import (
 )
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
-register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large")
 # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
 # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
 # (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly.

test/registered/kernels/benchmark/kv_canary/bench_write.py

@@ -30,7 +30,7 @@ from sglang.kernels.ops.kv_canary.verify import (
 from sglang.kernels.ops.kv_canary.write import WritePlan, launch_canary_write_kernel
 from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
 
-register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large")
 # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
 # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
 # (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly.

test/registered/lora/test_lora_deepseek_v3_base_logprob_diff.py

@@ -34,7 +34,7 @@ import sglang as sgl
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=1800, stage="nightly", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=1800, stage="weekly", runner_config="8-gpu-b200")
 
 BASE_MODEL = "deepseek-ai/DeepSeek-V3.1-Base"
 LORA_HF_REPO = "yushengsu/lora-diff-DeepSeek-V3.1-Base"

test/registered/lora/test_lora_kimi_k25_logprob_diff.py

@@ -34,7 +34,7 @@ import sglang as sgl
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=420, stage="nightly", runner_config="8-gpu-b200")
+register_cuda_ci(est_time=420, stage="weekly", runner_config="8-gpu-b200")
 
 BASE_MODEL = "moonshotai/Kimi-K2.5"
 LORA_HF_REPO = "yushengsu/lora-diff-Kimi-K2.5"

test/registered/lora/test_lora_openai_compatible.py

@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_cuda_ci(est_time=180, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=180, stage="weekly", runner_config="1-gpu-large")
 
 
 def setup_class(cls, *, enable_lora):

test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py

@@ -4,7 +4,7 @@ import sglang as sgl
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=300, stage="nightly", runner_config="4-gpu-h100")
+register_cuda_ci(est_time=300, stage="weekly", runner_config="4-gpu-h100")
 
 PROMPTS = [
     "Hello, my name is",

test/registered/model_loading/test_runai_model_loader.py

@@ -5,7 +5,7 @@ from sglang.srt.utils.common import temp_set_env
 from sglang.test.ci.ci_register import register_cuda_ci
 from sglang.test.test_utils import CustomTestCase
 
-register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=50, stage="weekly", runner_config="1-gpu-large")
 
 TEST_GCS_MODEL = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
 

test/registered/scheduler/test_routing_key_scheduling.py

@@ -20,7 +20,7 @@ from sglang.test.test_utils import (
     popen_launch_server,
 )
 
-register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large")
+register_cuda_ci(est_time=50, stage="weekly", runner_config="1-gpu-large")
 register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True)
 register_cpu_ci(est_time=55, suite="base-c-test-cpu")
 

test/registered/utils/test_model_file_verifier.py

@@ -1,349 +0,0 @@
-import hashlib
-import json
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import unittest
-import warnings
-from contextlib import nullcontext
-from io import StringIO
-
-import requests
-from huggingface_hub import snapshot_download
-
-from sglang.srt.utils import kill_process_tree
-from sglang.srt.utils.model_file_verifier import (
-    IntegrityError,
-    compute_sha256,
-    generate_checksums,
-    verify,
-)
-from sglang.test.ci.ci_register import register_cpu_ci
-from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
-    popen_launch_server,
-)
-
-# Note: AMD registration removed - test_model_file_verifier fails on AMD
-register_cpu_ci(est_time=146, suite="base-c-test-cpu")
-
-MODEL_NAME = "Qwen/Qwen3-0.6B"
-
-
-# ======== Base Test Classes ========
-
-
-class _FakeModelTestCase(unittest.TestCase):
-
-    FAKE_FILES = {
-        "model.safetensors": b"fake safetensors content " * 100,
-        "config.json": b'{"model_type": "llama"}',
-        "tokenizer.json": b'{"version": "1.0"}',
-    }
-
-    def setUp(self):
-        self.test_dir = tempfile.mkdtemp()
-        for filename, content in self.FAKE_FILES.items():
-            _create_test_file(self.test_dir, filename, content)
-
-    def tearDown(self):
-        shutil.rmtree(self.test_dir, ignore_errors=True)
-
-
-class _RealModelTestCase(unittest.TestCase):
-
-    @classmethod
-    def setUpClass(cls):
-        cls.original_model_path = snapshot_download(MODEL_NAME)
-
-    def setUp(self):
-        self.test_dir = tempfile.mkdtemp()
-        shutil.copytree(self.original_model_path, self.test_dir, dirs_exist_ok=True)
-
-    def tearDown(self):
-        shutil.rmtree(self.test_dir, ignore_errors=True)
-
-
-# ======== Unit Tests ========
-
-
-class TestModelFileVerifier(_FakeModelTestCase):
-
-    def test_detect_bit_rot(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        generate_checksums(source=self.test_dir, output_path=checksums_file)
-
-        target_file = os.path.join(self.test_dir, "model.safetensors")
-        _flip_bit_in_file(target_file, byte_offset=50, bit_position=3)
-
-        with self.assertRaises(IntegrityError) as ctx:
-            verify(model_path=self.test_dir, checksums_source=checksums_file)
-
-        self.assertIn("model.safetensors", str(ctx.exception))
-        self.assertIn("mismatch", str(ctx.exception).lower())
-
-    def test_detect_missing_file(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        generate_checksums(source=self.test_dir, output_path=checksums_file)
-
-        os.remove(os.path.join(self.test_dir, "config.json"))
-
-        with self.assertRaises(IntegrityError) as ctx:
-            verify(model_path=self.test_dir, checksums_source=checksums_file)
-
-        self.assertIn("config.json", str(ctx.exception))
-
-    def test_compute_sha256(self):
-        test_file = os.path.join(self.test_dir, "test.bin")
-        content = b"hello world"
-        with open(test_file, "wb") as f:
-            f.write(content)
-
-        result = compute_sha256(file_path=test_file)
-        expected = hashlib.sha256(content).hexdigest()
-        self.assertEqual(result, expected)
-
-    def test_parallel_checksum_computation(self):
-        for i in range(10):
-            _create_test_file(
-                self.test_dir, f"shard_{i}.safetensors", f"content_{i}".encode() * 1000
-            )
-
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        result = generate_checksums(
-            source=self.test_dir, output_path=checksums_file, max_workers=4
-        )
-
-        self.assertGreaterEqual(len(result.files), 10)
-
-    def test_generated_json_snapshot(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        generate_checksums(source=self.test_dir, output_path=checksums_file)
-
-        with open(checksums_file) as f:
-            data = json.load(f)
-
-        expected = {
-            "files": {
-                "config.json": {
-                    "sha256": "81dddc8c379baae137d99d24c5fa081d3a5ce52b6a221ddc22fe364711f8beaf",
-                    "size": 23,
-                },
-                "model.safetensors": {
-                    "sha256": "eb0c73a48a89fefb6b68dd41af830d75610c885135eac99139373b04705d05f3",
-                    "size": 2500,
-                },
-                "tokenizer.json": {
-                    "sha256": "4e3043229142b64d998563bc543ce034e0a2251af5d404995e3afcb8ce8850df",
-                    "size": 18,
-                },
-            }
-        }
-        self.assertEqual(data, expected)
-
-    def test_legacy_checksums_format_deprecated(self):
-        legacy_data = {
-            "checksums": {
-                "model.safetensors": "eb0c73a48a89fefb6b68dd41af830d75610c885135eac99139373b04705d05f3",
-                "config.json": "81dddc8c379baae137d99d24c5fa081d3a5ce52b6a221ddc22fe364711f8beaf",
-                "tokenizer.json": "4e3043229142b64d998563bc543ce034e0a2251af5d404995e3afcb8ce8850df",
-            }
-        }
-        legacy_file = os.path.join(self.test_dir, "legacy_checksums.json")
-        with open(legacy_file, "w") as f:
-            json.dump(legacy_data, f)
-
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter("always")
-            verify(model_path=self.test_dir, checksums_source=legacy_file)
-            self.assertEqual(len(w), 1)
-            self.assertTrue(issubclass(w[0].category, DeprecationWarning))
-            self.assertIn("deprecated", str(w[0].message).lower())
-
-
-# ======== CLI Tests ========
-
-
-class TestModelFileVerifierCLI(_FakeModelTestCase):
-
-    def test_cli_generate(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        result = subprocess.run(
-            [
-                sys.executable,
-                "-m",
-                "sglang.srt.utils.model_file_verifier",
-                "generate",
-                "--model-path",
-                self.test_dir,
-                "--model-checksum",
-                checksums_file,
-            ],
-            capture_output=True,
-            text=True,
-        )
-        self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
-        self.assertTrue(os.path.exists(checksums_file))
-
-        with open(checksums_file) as f:
-            data = json.load(f)
-        self.assertIn("files", data)
-        self.assertEqual(len(data["files"]), 3)
-
-    def test_cli_verify_success(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        generate_checksums(source=self.test_dir, output_path=checksums_file)
-
-        result = subprocess.run(
-            [
-                sys.executable,
-                "-m",
-                "sglang.srt.utils.model_file_verifier",
-                "verify",
-                "--model-path",
-                self.test_dir,
-                "--model-checksum",
-                checksums_file,
-            ],
-            capture_output=True,
-            text=True,
-        )
-        self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
-        self.assertIn("verified successfully", result.stdout)
-
-    def test_cli_verify_fails_on_corruption(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        generate_checksums(source=self.test_dir, output_path=checksums_file)
-
-        target_file = os.path.join(self.test_dir, "model.safetensors")
-        _flip_bit_in_file(target_file, byte_offset=50, bit_position=3)
-
-        result = subprocess.run(
-            [
-                sys.executable,
-                "-m",
-                "sglang.srt.utils.model_file_verifier",
-                "verify",
-                "--model-path",
-                self.test_dir,
-                "--model-checksum",
-                checksums_file,
-            ],
-            capture_output=True,
-            text=True,
-        )
-        self.assertNotEqual(result.returncode, 0)
-        combined = result.stdout + result.stderr
-        self.assertTrue(
-            "IntegrityError" in combined or "mismatch" in combined.lower(),
-            f"Expected integrity error, got: {combined}",
-        )
-
-
-# ======== HuggingFace Tests ========
-
-
-class TestModelFileVerifierHF(_RealModelTestCase):
-
-    def test_generate_checksums_from_hf(self):
-        checksums_file = os.path.join(self.test_dir, "checksums.json")
-        result = generate_checksums(source=MODEL_NAME, output_path=checksums_file)
-
-        self.assertTrue(os.path.exists(checksums_file))
-        self.assertGreater(len(result.files), 0)
-        for filename, file_info in result.files.items():
-            self.assertEqual(len(file_info.sha256), 64)
-
-    def test_verify_with_hf_checksums_source(self):
-        verify(model_path=self.test_dir, checksums_source=MODEL_NAME)
-
-
-# ======== Real Model E2E Tests ========
-
-
-class TestModelFileVerifierWithRealModel(_RealModelTestCase):
-
-    def _run_server_test(self, *, corrupt_weights: bool, use_hf_checksum: bool):
-        if use_hf_checksum:
-            checksum_arg = MODEL_NAME
-        else:
-            checksums_file = os.path.join(self.test_dir, "checksums.json")
-            generate_checksums(source=self.test_dir, output_path=checksums_file)
-            checksum_arg = checksums_file
-
-        corrupted_file = None
-        if corrupt_weights:
-            safetensors_files = [
-                f for f in os.listdir(self.test_dir) if f.endswith(".safetensors")
-            ]
-            self.assertTrue(len(safetensors_files) > 0, "No safetensors files found")
-            corrupted_file = safetensors_files[0]
-            _flip_bit_in_file(os.path.join(self.test_dir, corrupted_file))
-
-        stdout_io, stderr_io = StringIO(), StringIO()
-        ctx = self.assertRaises(Exception) if corrupt_weights else nullcontext()
-        with ctx:
-            process = popen_launch_server(
-                model=self.test_dir,
-                base_url=DEFAULT_URL_FOR_TEST,
-                timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-                other_args=["--model-checksum", checksum_arg],
-                return_stdout_stderr=(stdout_io, stderr_io),
-            )
-
-        if corrupt_weights:
-            output = stdout_io.getvalue() + stderr_io.getvalue()
-            self.assertIn(corrupted_file, output)
-            self.assertIn("mismatch", output.lower())
-        else:
-            try:
-                response = requests.post(
-                    f"{DEFAULT_URL_FOR_TEST}/generate",
-                    json={"text": "Hello", "sampling_params": {"max_new_tokens": 8}},
-                )
-                self.assertEqual(response.status_code, 200)
-                self.assertIn("text", response.json())
-            finally:
-                kill_process_tree(process.pid)
-
-    def test_server_launch_with_checksum_intact(self):
-        self._run_server_test(corrupt_weights=False, use_hf_checksum=False)
-
-    def test_server_launch_fails_with_corrupted_weights(self):
-        self._run_server_test(corrupt_weights=True, use_hf_checksum=False)
-
-    def test_server_launch_with_hf_checksum_intact(self):
-        self._run_server_test(corrupt_weights=False, use_hf_checksum=True)
-
-    def test_server_launch_with_hf_checksum_corrupted(self):
-        self._run_server_test(corrupt_weights=True, use_hf_checksum=True)
-
-
-# ======== Test Utilities ========
-
-
-def _create_test_file(directory: str, filename: str, content: bytes) -> str:
-    path = os.path.join(directory, filename)
-    with open(path, "wb") as f:
-        f.write(content)
-    return path
-
-
-def _flip_bit_in_file(file_path: str, byte_offset: int = 100, bit_position: int = 0):
-    file_size = os.path.getsize(file_path)
-    assert (
-        byte_offset < file_size
-    ), f"byte_offset {byte_offset} >= file_size {file_size}"
-
-    with open(file_path, "r+b") as f:
-        f.seek(byte_offset)
-        original_byte = f.read(1)[0]
-        f.seek(byte_offset)
-        f.write(bytes([original_byte ^ (1 << bit_position)]))
-
-
-if __name__ == "__main__":
-    unittest.main()

test/run_suite.py

@@ -188,12 +188,20 @@ NIGHTLY_SUITES = {
 OTHER_SUITES = {
     HWBackend.CPU: [
         "default",
+        # `stage="weekly"`, dispatched by weekly-test-cpu.yml.
+        "weekly-test-cpu",
     ],
     HWBackend.CUDA: [
         "stress",
         # `stage="weekly"` -- same shape. The three dicts group names for
-        # readability only; validation reads their union.
+        # readability only; validation reads their union. One entry per row of
+        # the matrix in weekly-test-nvidia.yml.
+        "weekly-test-1-gpu-large",
+        "weekly-test-2-gpu-large",
+        "weekly-test-4-gpu-h100",
+        "weekly-test-4-gpu-b200",
         "weekly-test-8-gpu-h200",
+        "weekly-test-8-gpu-b200",
     ],
 }