diff --git a/.github/workflows/_pr-test-stage.yml b/.github/workflows/_pr-test-stage.yml index 1d961be21..c29402e0b 100644 --- a/.github/workflows/_pr-test-stage.yml +++ b/.github/workflows/_pr-test-stage.yml @@ -101,9 +101,17 @@ jobs: name: ${{ inputs.self_name }} (${{ matrix.partition }}) # Runs on schedule / parallel-dispatch / non-failed PR with main_package or sgl_kernel changes. # Temporarily skip the broken GB300 runner before GitHub tries to allocate it. + # `runner_filter` is matched here rather than in each caller's job `if`: a + # caller that declares its runners as a matrix cannot, since job-level `if` + # sees github/needs/vars/inputs but not `matrix`. Callers that never declare + # the input read back null and match every runner_config. if: | always() && inputs.runner_config != '4-gpu-gb300' && + (fromJson(inputs.caller_inputs).runner_filter == null || + fromJson(inputs.caller_inputs).runner_filter == '' || + fromJson(inputs.caller_inputs).runner_filter == 'all' || + fromJson(inputs.caller_inputs).runner_filter == inputs.runner_config) && ((github.event_name == 'schedule' || fromJson(inputs.caller_inputs).test_parallel_dispatch == true) || (!failure() && !cancelled())) && (fromJson(inputs.check_changes).main_package == 'true' || fromJson(inputs.check_changes).sgl_kernel == 'true') # runs-on resolved from runs_on_map; check-changes already substituted diff --git a/.github/workflows/weekly-test-cpu.yml b/.github/workflows/weekly-test-cpu.yml new file mode 100644 index 000000000..77e5cd6a5 --- /dev/null +++ b/.github/workflows/weekly-test-cpu.yml @@ -0,0 +1,49 @@ +# Weekly CPU tests. Holds the CPU-only unit tests that are not worth a +# per-commit slot -- debug tooling (dump comparator, source patcher) whose +# breakage is fine to notice once a week. +# +# Separate from weekly-test-nvidia.yml because CPU goes through a different +# reusable workflow, and `uses:` takes no expressions -- so it cannot be another +# row in that file's matrix. Both run the same shared stage their per-commit +# counterparts do, so a test behaves identically in a PR and in the weekly run. +name: Weekly Test (CPU) + +on: + schedule: + - cron: '0 0 * * 0' # Sunday 00:00 UTC, alongside the Nvidia weekly run + workflow_dispatch: + +concurrency: + group: weekly-test-cpu-${{ github.ref }} + cancel-in-progress: true + +permissions: + actions: write + contents: read + issues: read + pull-requests: read + +jobs: + # run_all_tests skips the paths-filter, so every test runs regardless of what + # the last commit touched. + check-changes: + uses: ./.github/workflows/_pr-test-check-changes.yml + with: + pr_test_yml: '.github/workflows/weekly-test-cpu.yml' + run_all_tests: true + force_continue_on_error: true + secrets: inherit + + # No rust_ext_artifact: nothing builds one here, so the stage falls back to + # its cache and compiles on a miss. + weekly-test-cpu: + needs: check-changes + if: github.repository == 'sgl-project/sglang' + uses: ./.github/workflows/_pr-test-stage-cpu.yml + with: + self_name: weekly-test-cpu + check_changes: ${{ toJson(needs.check-changes.outputs) }} + caller_inputs: ${{ toJson(inputs) }} + partitions: ${{ needs.check-changes.outputs.partitions }} + run_timeout_minutes: '90' + secrets: inherit diff --git a/.github/workflows/weekly-test-nvidia.yml b/.github/workflows/weekly-test-nvidia.yml index 416a3a1d8..6fc23124d 100644 --- a/.github/workflows/weekly-test-nvidia.yml +++ b/.github/workflows/weekly-test-nvidia.yml @@ -8,13 +8,13 @@ on: workflow_dispatch: inputs: runner_filter: - description: 'Select which runner_config to run (leave empty or "all" to run all)' + # A free-text field rather than a choice: `on:` blocks take no + # expressions, so an options list would be a second copy of the matrix + # below to keep in sync. Matched in _pr-test-stage.yml. + description: 'runner_config to run alone (empty or "all" runs every one)' required: false - type: choice + type: string default: 'all' - options: - - 'all' - - '8-gpu-h200' full_parallel: description: 'Run all shards of a job at once (faster, but competes with per-commit CI for machines). Off by default: one shard at a time.' required: false @@ -43,17 +43,33 @@ jobs: force_continue_on_error: true secrets: inherit - weekly-test-8-gpu-h200: + # One row per machine type; timeouts mirror the same runner's nightly job. + # Adding a machine is one row here, and adding a test to an existing machine + # is none -- `stage="weekly"` + `runner_config=` on the test is enough. + weekly-test: + # Without this, the matrix default names the job after every value in the + # row -- "weekly-test (1-gpu-large, 120, 180)". + name: weekly-test-${{ matrix.runner_config }} needs: check-changes - if: github.repository == 'sgl-project/sglang' && (inputs.runner_filter == '' || inputs.runner_filter == 'all' || inputs.runner_filter == '8-gpu-h200') + if: github.repository == 'sgl-project/sglang' + strategy: + fail-fast: false + matrix: + include: + - { runner_config: 1-gpu-large, run_timeout: '120', job_timeout: '180' } + - { runner_config: 2-gpu-large, run_timeout: '240', job_timeout: '300' } + - { runner_config: 4-gpu-h100, run_timeout: '120', job_timeout: '180' } + - { runner_config: 4-gpu-b200, run_timeout: '150', job_timeout: '210' } + - { runner_config: 8-gpu-h200, run_timeout: '240', job_timeout: '300' } + - { runner_config: 8-gpu-b200, run_timeout: '360', job_timeout: '420' } uses: ./.github/workflows/_pr-test-stage.yml with: - self_name: weekly-test-8-gpu-h200 - runner_config: 8-gpu-h200 + self_name: weekly-test-${{ matrix.runner_config }} + runner_config: ${{ matrix.runner_config }} check_changes: ${{ toJson(needs.check-changes.outputs) }} caller_inputs: ${{ toJson(inputs) }} partitions: ${{ needs.check-changes.outputs.partitions }} - run_timeout_minutes: '240' - job_timeout_minutes: '300' + run_timeout_minutes: ${{ matrix.run_timeout }} + job_timeout_minutes: ${{ matrix.job_timeout }} scheduled: true secrets: inherit diff --git a/python/sglang/test/test_utils.py b/python/sglang/test/test_utils.py index aefa11251..143e9f641 100644 --- a/python/sglang/test/test_utils.py +++ b/python/sglang/test/test_utils.py @@ -147,12 +147,13 @@ DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST = "Barrrrry/DeepSeek-R1-W4AFP8" DEFAULT_ENABLE_ROUTED_EXPERTS_MODEL_NAME_FOR_TEST = "Qwen/Qwen3-30B-A3B" # Nightly tests -DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = ( - "meta-llama/Llama-3.1-8B-Instruct,Qwen/Qwen3-8B,Qwen/Qwen3-4B" +# Deliberate omission: a model another registered suite already uses as its base +# model is left out, since a regression there surfaces in that suite instead. +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = ( + "meta-llama/Llama-3.1-70B-Instruct,Qwen/Qwen2-57B-A14B-Instruct" ) -DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = "meta-llama/Llama-3.1-70B-Instruct,mistralai/Mixtral-8x7B-Instruct-v0.1,Qwen/Qwen2-57B-A14B-Instruct" -DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8,neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8" -DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,zai-org/GLM-4.5-Air-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1 = "neuralmagic/Mistral-7B-Instruct-v0.3-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8,neuralmagic/gemma-2-2b-it-FP8" +DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2 = "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8,neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8,neuralmagic/Qwen2-72B-Instruct-FP8,neuralmagic/Qwen2-57B-A14B-Instruct-FP8,neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8" DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1 = "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4,hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4,hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4" DEFAULT_SMALL_MODEL_NAME_FOR_TEST_QWEN = "Qwen/Qwen2.5-1.5B-Instruct" DEFAULT_SMALL_VLM_MODEL_NAME_FOR_TEST = "Qwen/Qwen2.5-VL-3B-Instruct" diff --git a/scripts/ci/utils/compute_partitions.py b/scripts/ci/utils/compute_partitions.py index a89bb341a..ab73ff00e 100644 --- a/scripts/ci/utils/compute_partitions.py +++ b/scripts/ci/utils/compute_partitions.py @@ -11,6 +11,7 @@ import importlib.util import json import math import os +import re from collections import defaultdict from datetime import datetime, timedelta @@ -49,6 +50,18 @@ _REUSABLE_STAGE_USES = { } +_MATRIX_REF = re.compile(r"\$\{\{\s*matrix\.([A-Za-z_][A-Za-z0-9_]*)\s*\}\}") + + +def _resolve_matrix_refs(value, row: dict): + """Substitute `${{ matrix.key }}` from one `strategy.matrix.include` row. + + A caller may declare its runners as a matrix instead of one job each, which + leaves `self_name` and `run_timeout_minutes` as expressions here -- GitHub + resolves them at dispatch, this static read has to do it itself.""" + return _MATRIX_REF.sub(lambda m: str(row[m.group(1)]), str(value)) + + def load_run_timeouts(pr_test_yml_path: str) -> dict: """Map `self_name -> run_timeout_minutes` from one pr-test*.yml. The input is required in both reusable stage workflows -- KeyError surfaces missing.""" @@ -60,8 +73,17 @@ def load_run_timeouts(pr_test_yml_path: str) -> dict: if not isinstance(job, dict) or job.get("uses") not in _REUSABLE_STAGE_USES: continue with_ = job.get("with") or {} - suite = with_.get("self_name", job_id) - timeouts[suite] = int(with_["run_timeout_minutes"]) + rows = ((job.get("strategy") or {}).get("matrix") or {}).get("include") or [ + None + ] + for row in rows: + if row is None: + suite = with_.get("self_name", job_id) + timeout = with_["run_timeout_minutes"] + else: + suite = _resolve_matrix_refs(with_.get("self_name", job_id), row) + timeout = _resolve_matrix_refs(with_["run_timeout_minutes"], row) + timeouts[suite] = int(timeout) if not timeouts: raise RuntimeError( f"load_run_timeouts: no jobs matched uses in {_REUSABLE_STAGE_USES!r} " diff --git a/test/registered/8-gpu-models/test_glm_46.py b/test/registered/8-gpu-models/test_glm_46.py index 2d3bc0b08..ca70b28fc 100644 --- a/test/registered/8-gpu-models/test_glm_46.py +++ b/test/registered/8-gpu-models/test_glm_46.py @@ -7,8 +7,8 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200: registered once per runner_config below -register_cuda_ci(est_time=1320, stage="nightly", runner_config="8-gpu-h200") -register_cuda_ci(est_time=1320, stage="nightly", runner_config="8-gpu-b200") +register_cuda_ci(est_time=1320, stage="weekly", runner_config="8-gpu-h200") +register_cuda_ci(est_time=1320, stage="weekly", runner_config="8-gpu-b200") GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6" diff --git a/test/registered/8-gpu-models/test_minimax_m25.py b/test/registered/8-gpu-models/test_minimax_m25.py index 22fb25d81..900a71b77 100644 --- a/test/registered/8-gpu-models/test_minimax_m25.py +++ b/test/registered/8-gpu-models/test_minimax_m25.py @@ -7,8 +7,8 @@ from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings # Runs on both H200 and B200: registered once per runner_config below -register_cuda_ci(est_time=1860, stage="nightly", runner_config="8-gpu-h200") -register_cuda_ci(est_time=1860, stage="nightly", runner_config="8-gpu-b200") +register_cuda_ci(est_time=1860, stage="weekly", runner_config="8-gpu-h200") +register_cuda_ci(est_time=1860, stage="weekly", runner_config="8-gpu-b200") MINIMAX_M25_MODEL_PATH = "MiniMaxAI/MiniMax-M2.5" diff --git a/test/registered/8-gpu-models/test_mistral_large3.py b/test/registered/8-gpu-models/test_mistral_large3.py index 0531f8f93..5e9341efb 100644 --- a/test/registered/8-gpu-models/test_mistral_large3.py +++ b/test/registered/8-gpu-models/test_mistral_large3.py @@ -10,8 +10,8 @@ from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system # Runs on both H200 and B200: registered once per runner_config below # Note: trtllm_mla backend may have hardware-specific behavior -register_cuda_ci(est_time=3000, stage="nightly", runner_config="8-gpu-h200") -register_cuda_ci(est_time=3000, stage="nightly", runner_config="8-gpu-b200") +register_cuda_ci(est_time=3000, stage="weekly", runner_config="8-gpu-h200") +register_cuda_ci(est_time=3000, stage="weekly", runner_config="8-gpu-b200") MISTRAL_LARGE3_FP8_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512" MISTRAL_LARGE3_NVFP4_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4" diff --git a/test/registered/8-gpu-models/test_ring_2_5_1t.py b/test/registered/8-gpu-models/test_ring_2_5_1t.py index e204fb7ac..ab58083a5 100644 --- a/test/registered/8-gpu-models/test_ring_2_5_1t.py +++ b/test/registered/8-gpu-models/test_ring_2_5_1t.py @@ -5,8 +5,8 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_combined_tests import run_combined_tests from sglang.test.test_utils import ModelLaunchSettings -register_cuda_ci(est_time=1680, stage="nightly", runner_config="8-gpu-h200") -register_cuda_ci(est_time=1680, stage="nightly", runner_config="8-gpu-b200") +register_cuda_ci(est_time=1680, stage="weekly", runner_config="8-gpu-h200") +register_cuda_ci(est_time=1680, stage="weekly", runner_config="8-gpu-b200") RING_2_5_1T_MODEL_PATH = "inclusionAI/Ring-2.5-1T" diff --git a/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py index 9a37ed6d5..ed8ceb739 100644 --- a/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py @@ -20,7 +20,6 @@ from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, - DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, @@ -80,9 +79,6 @@ def remove_failing_models(model_str): return ",".join(filtered) -DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models( - DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 -) DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models( DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 ) @@ -206,7 +202,6 @@ class TestNightlyGsm8KEval(unittest.TestCase): @classmethod def setUpClass(cls): cls.model_groups = [ - (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False), (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True), (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False), (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True), diff --git a/test/registered/attention/test_deepseek_v3_deterministic.py b/test/registered/attention/test_deepseek_v3_deterministic.py index d38a7f799..3c9e41e1f 100644 --- a/test/registered/attention/test_deepseek_v3_deterministic.py +++ b/test/registered/attention/test_deepseek_v3_deterministic.py @@ -12,7 +12,7 @@ from sglang.test.test_deterministic_utils import ( TestDeterministicBase, ) -register_cuda_ci(est_time=240, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=240, stage="weekly", runner_config="1-gpu-large") DEEPSEEK_MODEL = "lmsys/sglang-ci-dsv3-test" diff --git a/test/registered/attention/test_qwen3_next_deterministic.py b/test/registered/attention/test_qwen3_next_deterministic.py index b276f362b..4278f2362 100644 --- a/test/registered/attention/test_qwen3_next_deterministic.py +++ b/test/registered/attention/test_qwen3_next_deterministic.py @@ -12,7 +12,7 @@ from sglang.test.test_deterministic_utils import ( TestDeterministicBase, ) -register_cuda_ci(est_time=200, stage="nightly", runner_config="4-gpu-h100") +register_cuda_ci(est_time=200, stage="weekly", runner_config="4-gpu-h100") QWEN3_NEXT = "Qwen/Qwen3-Next-80B-A3B-Instruct" diff --git a/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py b/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py index cea8fa5e6..30f1e5c88 100644 --- a/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py +++ b/test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( write_github_step_summary, ) -register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-b200") +register_cuda_ci(est_time=900, stage="extra-b", runner_config="4-gpu-b200") FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" SERVER_LAUNCH_TIMEOUT = 1000 diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index 1f9d2a2ea..fa9a3f862 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=1770, stage="nightly", runner_config="4-gpu-b200") +register_cuda_ci(est_time=1770, stage="weekly", runner_config="4-gpu-b200") class FlashinferTrtllmGenMoeBackendFP8Base: diff --git a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py index b76c48111..ebb576870 100644 --- a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py +++ b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py @@ -11,7 +11,7 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase -register_cuda_ci(est_time=130, stage="nightly", runner_config="4-gpu-h100") +register_cuda_ci(est_time=130, stage="weekly", runner_config="4-gpu-h100") class TestPCGWithEAGLE3(PCGSpecBase, unittest.TestCase): diff --git a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_dflash.py b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_dflash.py index e52548d04..1a1a6412a 100644 --- a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_dflash.py +++ b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_dflash.py @@ -15,7 +15,7 @@ from sglang.test.test_utils import ( CustomTestCase, ) -register_cuda_ci(est_time=110, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=110, stage="weekly", runner_config="1-gpu-large") class TestPCGWithDFlash(PCGSpecBase, CustomTestCase): diff --git a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py index 5aa0c4dd7..fd6cd41cc 100644 --- a/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py +++ b/test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py @@ -8,7 +8,7 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase -register_cuda_ci(est_time=450, stage="nightly", runner_config="4-gpu-h100") +register_cuda_ci(est_time=450, stage="weekly", runner_config="4-gpu-h100") class TestPCGWithMTP(PCGSpecBase, unittest.TestCase): diff --git a/test/registered/debug_utils/comparator/aligner/entrypoint/test_executor.py b/test/registered/debug_utils/comparator/aligner/entrypoint/test_executor.py index 46d149a71..8276f6115 100644 --- a/test/registered/debug_utils/comparator/aligner/entrypoint/test_executor.py +++ b/test/registered/debug_utils/comparator/aligner/entrypoint/test_executor.py @@ -33,8 +33,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") class TestExecuteSubPlans: diff --git a/test/registered/debug_utils/comparator/aligner/entrypoint/test_planner.py b/test/registered/debug_utils/comparator/aligner/entrypoint/test_planner.py index 459df88cf..88207e50a 100644 --- a/test/registered/debug_utils/comparator/aligner/entrypoint/test_planner.py +++ b/test/registered/debug_utils/comparator/aligner/entrypoint/test_planner.py @@ -25,8 +25,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") def _make_meta( diff --git a/test/registered/debug_utils/comparator/aligner/reorderer/test_executor.py b/test/registered/debug_utils/comparator/aligner/reorderer/test_executor.py index 08e2e5c2d..b089aa23d 100644 --- a/test/registered/debug_utils/comparator/aligner/reorderer/test_executor.py +++ b/test/registered/debug_utils/comparator/aligner/reorderer/test_executor.py @@ -26,8 +26,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _zigzag_order(cp_size: int) -> list[int]: diff --git a/test/registered/debug_utils/comparator/aligner/reorderer/test_planner.py b/test/registered/debug_utils/comparator/aligner/reorderer/test_planner.py index cc4851cab..f0c9d9223 100644 --- a/test/registered/debug_utils/comparator/aligner/reorderer/test_planner.py +++ b/test/registered/debug_utils/comparator/aligner/reorderer/test_planner.py @@ -26,8 +26,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") class TestComputeReordererPlans: diff --git a/test/registered/debug_utils/comparator/aligner/test_axis_aligner.py b/test/registered/debug_utils/comparator/aligner/test_axis_aligner.py index 594e50cf2..7267e0510 100644 --- a/test/registered/debug_utils/comparator/aligner/test_axis_aligner.py +++ b/test/registered/debug_utils/comparator/aligner/test_axis_aligner.py @@ -17,8 +17,7 @@ from sglang.srt.debug_utils.comparator.log_sink import log_sink from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") class TestComputeAxisAlignerPlan: diff --git a/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_loader.py b/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_loader.py index cfecd18f2..24f754fe3 100644 --- a/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_loader.py +++ b/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_loader.py @@ -19,7 +19,7 @@ from sglang.srt.debug_utils.comparator.log_sink import LogSink from sglang.srt.debug_utils.comparator.output_types import ErrorLog, InfoLog from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") _sglang_plugin = _SGLangPlugin() _megatron_plugin = _MegatronPlugin() diff --git a/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_plugins.py b/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_plugins.py index 2621cd2f5..b61e7e559 100644 --- a/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_plugins.py +++ b/test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_plugins.py @@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.aligner.token_aligner.smart.types import from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") _sglang_plugin = _SGLangPlugin() _megatron_plugin = _MegatronPlugin() diff --git a/test/registered/debug_utils/comparator/aligner/token_aligner/test_concat_steps.py b/test/registered/debug_utils/comparator/aligner/token_aligner/test_concat_steps.py index fae03b4a2..6dbcc29f6 100644 --- a/test/registered/debug_utils/comparator/aligner/token_aligner/test_concat_steps.py +++ b/test/registered/debug_utils/comparator/aligner/token_aligner/test_concat_steps.py @@ -10,8 +10,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import apply_dim_names from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") class TestExecuteConcat: diff --git a/test/registered/debug_utils/comparator/aligner/token_aligner/test_executor.py b/test/registered/debug_utils/comparator/aligner/token_aligner/test_executor.py index a8f60dcf7..7bc2e4086 100644 --- a/test/registered/debug_utils/comparator/aligner/token_aligner/test_executor.py +++ b/test/registered/debug_utils/comparator/aligner/token_aligner/test_executor.py @@ -29,8 +29,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") def _named(tensor: torch.Tensor, names: list[str]) -> torch.Tensor: diff --git a/test/registered/debug_utils/comparator/aligner/token_aligner/test_planner.py b/test/registered/debug_utils/comparator/aligner/token_aligner/test_planner.py index 1e3758623..a1cb5d6e1 100644 --- a/test/registered/debug_utils/comparator/aligner/token_aligner/test_planner.py +++ b/test/registered/debug_utils/comparator/aligner/token_aligner/test_planner.py @@ -23,8 +23,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu") class TestBuildTokenIndexSGLangThd: diff --git a/test/registered/debug_utils/comparator/aligner/token_aligner/test_thd_seq_lens_loader.py b/test/registered/debug_utils/comparator/aligner/token_aligner/test_thd_seq_lens_loader.py index b5f348cf2..98336b3b7 100644 --- a/test/registered/debug_utils/comparator/aligner/token_aligner/test_thd_seq_lens_loader.py +++ b/test/registered/debug_utils/comparator/aligner/token_aligner/test_thd_seq_lens_loader.py @@ -14,7 +14,7 @@ from sglang.srt.debug_utils.comparator.aligner.token_aligner.smart.aux_plugins i ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") def _save_pt( diff --git a/test/registered/debug_utils/comparator/aligner/unsharder/test_executor.py b/test/registered/debug_utils/comparator/aligner/unsharder/test_executor.py index 7acd25be7..0bc7bec89 100644 --- a/test/registered/debug_utils/comparator/aligner/unsharder/test_executor.py +++ b/test/registered/debug_utils/comparator/aligner/unsharder/test_executor.py @@ -30,8 +30,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( from sglang.srt.debug_utils.comparator.output_types import ReplicatedCheckResult from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _name_tensors( diff --git a/test/registered/debug_utils/comparator/aligner/unsharder/test_parallel_info.py b/test/registered/debug_utils/comparator/aligner/unsharder/test_parallel_info.py index 7b23e3618..5908d6471 100644 --- a/test/registered/debug_utils/comparator/aligner/unsharder/test_parallel_info.py +++ b/test/registered/debug_utils/comparator/aligner/unsharder/test_parallel_info.py @@ -9,8 +9,7 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import AxisInfo from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") class TestNormalizeParallelInfo: diff --git a/test/registered/debug_utils/comparator/aligner/unsharder/test_planner.py b/test/registered/debug_utils/comparator/aligner/unsharder/test_planner.py index f2f6a52a4..25e0d73de 100644 --- a/test/registered/debug_utils/comparator/aligner/unsharder/test_planner.py +++ b/test/registered/debug_utils/comparator/aligner/unsharder/test_planner.py @@ -19,8 +19,7 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import ( from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis, parse_dims from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") class TestComputeUnsharderPlan: diff --git a/test/registered/debug_utils/comparator/dims_spec/test_dim_parser.py b/test/registered/debug_utils/comparator/dims_spec/test_dim_parser.py index e6f69b9df..0f9aec1f6 100644 --- a/test/registered/debug_utils/comparator/dims_spec/test_dim_parser.py +++ b/test/registered/debug_utils/comparator/dims_spec/test_dim_parser.py @@ -12,8 +12,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=6, stage="weekly", runner_config="cpu") class TestParseDim: diff --git a/test/registered/debug_utils/comparator/dims_spec/test_dims_parser.py b/test/registered/debug_utils/comparator/dims_spec/test_dims_parser.py index 7aa0b18da..7163e2e58 100644 --- a/test/registered/debug_utils/comparator/dims_spec/test_dims_parser.py +++ b/test/registered/debug_utils/comparator/dims_spec/test_dims_parser.py @@ -15,8 +15,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu") class TestSingletonDimUtilFilterOut: diff --git a/test/registered/debug_utils/comparator/dims_spec/test_tensor_naming.py b/test/registered/debug_utils/comparator/dims_spec/test_tensor_naming.py index cdebcf010..f51ab71f8 100644 --- a/test/registered/debug_utils/comparator/dims_spec/test_tensor_naming.py +++ b/test/registered/debug_utils/comparator/dims_spec/test_tensor_naming.py @@ -14,8 +14,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=7, stage="weekly", runner_config="cpu") class TestFindDimIndex: diff --git a/test/registered/debug_utils/comparator/dims_spec/test_types.py b/test/registered/debug_utils/comparator/dims_spec/test_types.py index 7f36aead0..e25c87262 100644 --- a/test/registered/debug_utils/comparator/dims_spec/test_types.py +++ b/test/registered/debug_utils/comparator/dims_spec/test_types.py @@ -9,8 +9,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu") class TestDimConstants: diff --git a/test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py b/test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py index 9a9e0c1fa..9b3b6b0af 100644 --- a/test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py +++ b/test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py @@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import DiffInfo from sglang.srt.debug_utils.comparator.threshold_dsl import DiffThresholdRule from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=20, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=20, stage="weekly", runner_config="cpu") class TestComputeTensorInfo: diff --git a/test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py b/test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py index 8dacb7721..d3c4534da 100644 --- a/test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py +++ b/test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py @@ -63,7 +63,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import ( from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") _DEFAULT_PERCENTILE_LINES: list[str] = [ " [blue]p1 [/] -1.8000 -1.8000 [dim]+0.00e+00[/]", diff --git a/test/registered/debug_utils/comparator/tensor_comparator/test_types.py b/test/registered/debug_utils/comparator/tensor_comparator/test_types.py index 46f19fbcf..8fef68c2a 100644 --- a/test/registered/debug_utils/comparator/tensor_comparator/test_types.py +++ b/test/registered/debug_utils/comparator/tensor_comparator/test_types.py @@ -21,8 +21,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _make_stats(**overrides) -> TensorStats: diff --git a/test/registered/debug_utils/comparator/test_bundle_comparator.py b/test/registered/debug_utils/comparator/test_bundle_comparator.py index 64e1ae041..b16262e4c 100644 --- a/test/registered/debug_utils/comparator/test_bundle_comparator.py +++ b/test/registered/debug_utils/comparator/test_bundle_comparator.py @@ -15,7 +15,7 @@ from sglang.srt.debug_utils.comparator.utils import Pair from sglang.srt.debug_utils.dump_loader import ValueWithMeta from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") def _save_tensor( diff --git a/test/registered/debug_utils/comparator/test_bundle_matcher.py b/test/registered/debug_utils/comparator/test_bundle_matcher.py index 586e8d6e0..3b94594f5 100644 --- a/test/registered/debug_utils/comparator/test_bundle_matcher.py +++ b/test/registered/debug_utils/comparator/test_bundle_matcher.py @@ -13,7 +13,7 @@ from sglang.srt.debug_utils.comparator.bundle_matcher import ( from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") def _make_row( diff --git a/test/registered/debug_utils/comparator/test_display.py b/test/registered/debug_utils/comparator/test_display.py index 1227dcb41..d8e2dc327 100644 --- a/test/registered/debug_utils/comparator/test_display.py +++ b/test/registered/debug_utils/comparator/test_display.py @@ -21,7 +21,7 @@ from sglang.srt.debug_utils.comparator.output_types import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _render_rich(renderable: object) -> str: diff --git a/test/registered/debug_utils/comparator/test_dp_utils.py b/test/registered/debug_utils/comparator/test_dp_utils.py index 70d484e60..627dc5fc5 100644 --- a/test/registered/debug_utils/comparator/test_dp_utils.py +++ b/test/registered/debug_utils/comparator/test_dp_utils.py @@ -12,7 +12,7 @@ from sglang.srt.debug_utils.comparator.dp_utils import ( from sglang.srt.debug_utils.dump_loader import ValueWithMeta from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu") def _make_sglang_meta( diff --git a/test/registered/debug_utils/comparator/test_dump_loader.py b/test/registered/debug_utils/comparator/test_dump_loader.py index 9b5a25bce..a79c2ca9b 100644 --- a/test/registered/debug_utils/comparator/test_dump_loader.py +++ b/test/registered/debug_utils/comparator/test_dump_loader.py @@ -7,7 +7,7 @@ import torch from sglang.srt.debug_utils.dump_loader import read_tokenizer_path from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _save_pt( diff --git a/test/registered/debug_utils/comparator/test_e2e_demo.py b/test/registered/debug_utils/comparator/test_e2e_demo.py index eb26749bb..fb7f13a4a 100644 --- a/test/registered/debug_utils/comparator/test_e2e_demo.py +++ b/test/registered/debug_utils/comparator/test_e2e_demo.py @@ -26,7 +26,7 @@ from sglang.srt.debug_utils.comparator.output_types import ( from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="default", nightly=True) +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") _EXP_NAME = "demo_exp" diff --git a/test/registered/debug_utils/comparator/test_entrypoint.py b/test/registered/debug_utils/comparator/test_entrypoint.py index 02c7de002..361b50c3f 100644 --- a/test/registered/debug_utils/comparator/test_entrypoint.py +++ b/test/registered/debug_utils/comparator/test_entrypoint.py @@ -30,7 +30,7 @@ from sglang.srt.debug_utils.comparator.output_types import ( from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper, _RecomputeStatus from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu") _FIXED_EXP_NAME = "my_exp_name" diff --git a/test/registered/debug_utils/comparator/test_log_sink.py b/test/registered/debug_utils/comparator/test_log_sink.py index 977f8fee3..fc990a536 100644 --- a/test/registered/debug_utils/comparator/test_log_sink.py +++ b/test/registered/debug_utils/comparator/test_log_sink.py @@ -11,8 +11,7 @@ from sglang.srt.debug_utils.comparator.output_types import ( from sglang.srt.debug_utils.comparator.report_sink import report_sink from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _make_error_log(**overrides) -> ErrorLog: diff --git a/test/registered/debug_utils/comparator/test_manually_verify.py b/test/registered/debug_utils/comparator/test_manually_verify.py index 2c15299bc..64decdaae 100644 --- a/test/registered/debug_utils/comparator/test_manually_verify.py +++ b/test/registered/debug_utils/comparator/test_manually_verify.py @@ -23,7 +23,7 @@ import torch from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=60, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=60, stage="weekly", runner_config="cpu") _PUBLISH_DIR: Path = Path("/tmp/comparator_manual_verify") _PNG_MAGIC: bytes = b"\x89PNG" diff --git a/test/registered/debug_utils/comparator/test_meta_overrider.py b/test/registered/debug_utils/comparator/test_meta_overrider.py index 4a42045a6..05d23f88f 100644 --- a/test/registered/debug_utils/comparator/test_meta_overrider.py +++ b/test/registered/debug_utils/comparator/test_meta_overrider.py @@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.meta_overrider import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") # ───────────────────── Unit: MetaOverrideRule ───────────────────── diff --git a/test/registered/debug_utils/comparator/test_model_validation.py b/test/registered/debug_utils/comparator/test_model_validation.py index 3d8ef5671..995c04421 100644 --- a/test/registered/debug_utils/comparator/test_model_validation.py +++ b/test/registered/debug_utils/comparator/test_model_validation.py @@ -44,8 +44,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import ( from sglang.srt.debug_utils.comparator.utils import Pair, _check_equal_lengths from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") class TestCheckEqualLengths: diff --git a/test/registered/debug_utils/comparator/test_output_types.py b/test/registered/debug_utils/comparator/test_output_types.py index ef1b8e537..d6eec78b4 100644 --- a/test/registered/debug_utils/comparator/test_output_types.py +++ b/test/registered/debug_utils/comparator/test_output_types.py @@ -60,7 +60,7 @@ from sglang.srt.debug_utils.comparator.output_types import ( from sglang.srt.debug_utils.comparator.utils import Pair from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _render_rich(renderable: object) -> str: diff --git a/test/registered/debug_utils/comparator/test_per_token_visualizer.py b/test/registered/debug_utils/comparator/test_per_token_visualizer.py index 467705c9d..659dff491 100644 --- a/test/registered/debug_utils/comparator/test_per_token_visualizer.py +++ b/test/registered/debug_utils/comparator/test_per_token_visualizer.py @@ -15,7 +15,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.comparator import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu") _PNG_MAGIC: bytes = b"\x89PNG" diff --git a/test/registered/debug_utils/comparator/test_preset.py b/test/registered/debug_utils/comparator/test_preset.py index 91b4a92dd..246da15b8 100644 --- a/test/registered/debug_utils/comparator/test_preset.py +++ b/test/registered/debug_utils/comparator/test_preset.py @@ -3,8 +3,7 @@ import pytest from sglang.srt.debug_utils.comparator.preset import PRESETS, expand_preset from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=8, suite="base-c-test-cpu") +register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu") class TestExpandPreset: diff --git a/test/registered/debug_utils/comparator/test_threshold_dsl.py b/test/registered/debug_utils/comparator/test_threshold_dsl.py index 1bbd80aa6..64986d9c5 100644 --- a/test/registered/debug_utils/comparator/test_threshold_dsl.py +++ b/test/registered/debug_utils/comparator/test_threshold_dsl.py @@ -11,7 +11,7 @@ from sglang.srt.debug_utils.comparator.threshold_dsl import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") def _ev( diff --git a/test/registered/debug_utils/comparator/test_utils.py b/test/registered/debug_utils/comparator/test_utils.py index dbf79755e..1ca919eea 100644 --- a/test/registered/debug_utils/comparator/test_utils.py +++ b/test/registered/debug_utils/comparator/test_utils.py @@ -17,8 +17,7 @@ from sglang.srt.debug_utils.comparator.utils import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=7, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") class TestCalcRelDiff: diff --git a/test/registered/debug_utils/comparator/test_visualizer.py b/test/registered/debug_utils/comparator/test_visualizer.py index 772d26477..b62d8e7de 100644 --- a/test/registered/debug_utils/comparator/test_visualizer.py +++ b/test/registered/debug_utils/comparator/test_visualizer.py @@ -10,7 +10,7 @@ from sglang.srt.debug_utils.comparator.visualizer.preprocessing import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu") class TestPreprocessTensor: diff --git a/test/registered/debug_utils/comparator/testing_helpers.py b/test/registered/debug_utils/comparator/testing_helpers.py index 41f39462b..24e66347f 100644 --- a/test/registered/debug_utils/comparator/testing_helpers.py +++ b/test/registered/debug_utils/comparator/testing_helpers.py @@ -12,8 +12,8 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci( est_time=0, - suite="base-a-test-cpu", - nightly=True, + stage="weekly", + runner_config="cpu", disabled="helper module, no tests", ) diff --git a/test/registered/debug_utils/source_patcher/test_code_patcher.py b/test/registered/debug_utils/source_patcher/test_code_patcher.py index 71330cb5b..c9c47cbe6 100644 --- a/test/registered/debug_utils/source_patcher/test_code_patcher.py +++ b/test/registered/debug_utils/source_patcher/test_code_patcher.py @@ -10,8 +10,7 @@ from sglang.srt.debug_utils.source_patcher.code_patcher import ( from sglang.srt.debug_utils.source_patcher.types import EditSpec, PatchSpec from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=5, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") SAMPLE_MODULE_NAME = "_source_patcher_test_fixtures.sample_module" diff --git a/test/registered/debug_utils/source_patcher/test_dumper_integration.py b/test/registered/debug_utils/source_patcher/test_dumper_integration.py index 7f47915ea..588aa2f03 100644 --- a/test/registered/debug_utils/source_patcher/test_dumper_integration.py +++ b/test/registered/debug_utils/source_patcher/test_dumper_integration.py @@ -8,8 +8,7 @@ import yaml from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=6, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") SAMPLE_MODULE_NAME = "_source_patcher_test_fixtures.sample_module" diff --git a/test/registered/debug_utils/source_patcher/test_source_editor.py b/test/registered/debug_utils/source_patcher/test_source_editor.py index e8471af86..38898e6d8 100644 --- a/test/registered/debug_utils/source_patcher/test_source_editor.py +++ b/test/registered/debug_utils/source_patcher/test_source_editor.py @@ -5,8 +5,7 @@ from sglang.srt.debug_utils.source_patcher.source_editor import apply_edits from sglang.srt.debug_utils.source_patcher.types import EditSpec, PatchApplicationError from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=4, suite="base-c-test-cpu") +register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu") class TestApplyEdits: diff --git a/test/registered/debug_utils/test_dump_comparator.py b/test/registered/debug_utils/test_dump_comparator.py index d269bee48..0715e3cc9 100644 --- a/test/registered/debug_utils/test_dump_comparator.py +++ b/test/registered/debug_utils/test_dump_comparator.py @@ -9,7 +9,7 @@ from sglang.srt.debug_utils.dump_comparator import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu") # ----------------------------- Unit tests ----------------------------- diff --git a/test/registered/debug_utils/test_dump_loader.py b/test/registered/debug_utils/test_dump_loader.py index bc6a73bfc..27e508801 100644 --- a/test/registered/debug_utils/test_dump_loader.py +++ b/test/registered/debug_utils/test_dump_loader.py @@ -15,7 +15,7 @@ from sglang.srt.debug_utils.dump_loader import ( ) from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True) +register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu") class TestReadMeta: diff --git a/test/registered/debug_utils/test_schedule_simulator.py b/test/registered/debug_utils/test_schedule_simulator.py index 2549e68df..d6ca6280b 100644 --- a/test/registered/debug_utils/test_schedule_simulator.py +++ b/test/registered/debug_utils/test_schedule_simulator.py @@ -25,8 +25,7 @@ from sglang.srt.debug_utils.schedule_simulator import ( from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=120, suite="base-a-test-cpu", nightly=True) -register_cpu_ci(est_time=357, suite="base-c-test-cpu") +register_cpu_ci(est_time=357, stage="weekly", runner_config="cpu") # ==================== Non-E2E Tests ==================== diff --git a/test/registered/eval/test_text_models_gsm8k_eval.py b/test/registered/eval/test_text_models_gsm8k_eval.py index db907175f..359ffe635 100644 --- a/test/registered/eval/test_text_models_gsm8k_eval.py +++ b/test/registered/eval/test_text_models_gsm8k_eval.py @@ -9,7 +9,6 @@ from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1, DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2, - DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1, DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2, DEFAULT_URL_FOR_TEST, ModelLaunchSettings, @@ -23,21 +22,15 @@ from sglang.test.test_utils import ( # downloading on cache miss. Use a longer timeout than the default 600s. NIGHTLY_EVAL_SERVER_TIMEOUT = 1800 -register_cuda_ci(est_time=2880, stage="nightly", runner_config="2-gpu-large") +register_cuda_ci(est_time=2880, stage="weekly", runner_config="2-gpu-large") MODEL_SCORE_THRESHOLDS = { # sgl-eval (zero-shot chat, \boxed{}, math_verify grading). Thresholds are # measured_score - 0.05, baselined on H100 2-GPU over the full 1319 split. - "meta-llama/Llama-3.1-8B-Instruct": 0.77, # 81.05% measured - 5% - "Qwen/Qwen3-8B": 0.76, # 81.43% measured - 5% - "Qwen/Qwen3-4B": 0.77, # 82.41% measured - 5% "meta-llama/Llama-3.1-70B-Instruct": 0.90, # 94.77% measured - 5% - "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.39, # 43.52% measured - 5% "Qwen/Qwen2-57B-A14B-Instruct": 0.46, # 50.87% measured - 5% - "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.77, # 82.34% measured - 5% "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.23, # 27.82% measured - 5% "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.80, # 84.91% measured - 5% - "zai-org/GLM-4.5-Air-FP8": 0.73, # 77.48% measured - 5% "neuralmagic/gemma-2-2b-it-FP8": 0.02, # 6.52% measured - 5% "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.01% measured - 5% "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.35, # 40.33% measured - 5% @@ -51,9 +44,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): @classmethod def setUpClass(cls): cls.models = [] - models_tp1 = parse_models( - DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 - ) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1) + models_tp1 = parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1) for model_path in models_tp1: cls.models.append(ModelLaunchSettings(model_path, tp_size=1)) diff --git a/test/registered/kernels/benchmark/kv_canary/bench_plan.py b/test/registered/kernels/benchmark/kv_canary/bench_plan.py index 9b952a994..59f1fef10 100644 --- a/test/registered/kernels/benchmark/kv_canary/bench_plan.py +++ b/test/registered/kernels/benchmark/kv_canary/bench_plan.py @@ -25,7 +25,7 @@ from sglang.kernels.ops.kv_canary.verify import VerifyPlan from sglang.kernels.ops.kv_canary.write import WritePlan from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -register_cuda_ci(est_time=40, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=40, stage="weekly", runner_config="1-gpu-large") # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True) diff --git a/test/registered/kernels/benchmark/kv_canary/bench_scatter_req_token_ids.py b/test/registered/kernels/benchmark/kv_canary/bench_scatter_req_token_ids.py index 159b1bc93..ff109c9b8 100644 --- a/test/registered/kernels/benchmark/kv_canary/bench_scatter_req_token_ids.py +++ b/test/registered/kernels/benchmark/kv_canary/bench_scatter_req_token_ids.py @@ -16,7 +16,7 @@ from sglang.kernels.ops.kv_canary.scatter_req_token_ids import ( ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large") # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range # (_BS_AXIS_CI/_SEQ_LEN_AXIS_CI via get_benchmark_range), same as CUDA nightly. diff --git a/test/registered/kernels/benchmark/kv_canary/bench_verify.py b/test/registered/kernels/benchmark/kv_canary/bench_verify.py index 2d81fd8ef..36d33e014 100644 --- a/test/registered/kernels/benchmark/kv_canary/bench_verify.py +++ b/test/registered/kernels/benchmark/kv_canary/bench_verify.py @@ -32,7 +32,7 @@ from sglang.kernels.ops.kv_canary.verify import ( ) from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large") # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range # (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly. diff --git a/test/registered/kernels/benchmark/kv_canary/bench_write.py b/test/registered/kernels/benchmark/kv_canary/bench_write.py index 71b22e33f..da8fdd962 100644 --- a/test/registered/kernels/benchmark/kv_canary/bench_write.py +++ b/test/registered/kernels/benchmark/kv_canary/bench_write.py @@ -30,7 +30,7 @@ from sglang.kernels.ops.kv_canary.verify import ( from sglang.kernels.ops.kv_canary.write import WritePlan, launch_canary_write_kernel from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large") # AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite). # Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range # (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly. diff --git a/test/registered/lora/test_lora_deepseek_v3_base_logprob_diff.py b/test/registered/lora/test_lora_deepseek_v3_base_logprob_diff.py index f73fcc4da..c7a93597e 100644 --- a/test/registered/lora/test_lora_deepseek_v3_base_logprob_diff.py +++ b/test/registered/lora/test_lora_deepseek_v3_base_logprob_diff.py @@ -34,7 +34,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=1800, stage="nightly", runner_config="8-gpu-b200") +register_cuda_ci(est_time=1800, stage="weekly", runner_config="8-gpu-b200") BASE_MODEL = "deepseek-ai/DeepSeek-V3.1-Base" LORA_HF_REPO = "yushengsu/lora-diff-DeepSeek-V3.1-Base" diff --git a/test/registered/lora/test_lora_kimi_k25_logprob_diff.py b/test/registered/lora/test_lora_kimi_k25_logprob_diff.py index a62348aaf..891997b7f 100644 --- a/test/registered/lora/test_lora_kimi_k25_logprob_diff.py +++ b/test/registered/lora/test_lora_kimi_k25_logprob_diff.py @@ -34,7 +34,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=420, stage="nightly", runner_config="8-gpu-b200") +register_cuda_ci(est_time=420, stage="weekly", runner_config="8-gpu-b200") BASE_MODEL = "moonshotai/Kimi-K2.5" LORA_HF_REPO = "yushengsu/lora-diff-Kimi-K2.5" diff --git a/test/registered/lora/test_lora_openai_compatible.py b/test/registered/lora/test_lora_openai_compatible.py index 24b163d33..4fcf9a26f 100644 --- a/test/registered/lora/test_lora_openai_compatible.py +++ b/test/registered/lora/test_lora_openai_compatible.py @@ -14,7 +14,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=180, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=180, stage="weekly", runner_config="1-gpu-large") def setup_class(cls, *, enable_lora): diff --git a/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py b/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py index b1778fb04..373a293c2 100644 --- a/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py +++ b/test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py @@ -4,7 +4,7 @@ import sglang as sgl from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=300, stage="nightly", runner_config="4-gpu-h100") +register_cuda_ci(est_time=300, stage="weekly", runner_config="4-gpu-h100") PROMPTS = [ "Hello, my name is", diff --git a/test/registered/model_loading/test_runai_model_loader.py b/test/registered/model_loading/test_runai_model_loader.py index e830671d2..98a24b290 100644 --- a/test/registered/model_loading/test_runai_model_loader.py +++ b/test/registered/model_loading/test_runai_model_loader.py @@ -5,7 +5,7 @@ from sglang.srt.utils.common import temp_set_env from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.test_utils import CustomTestCase -register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=50, stage="weekly", runner_config="1-gpu-large") TEST_GCS_MODEL = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/" diff --git a/test/registered/scheduler/test_routing_key_scheduling.py b/test/registered/scheduler/test_routing_key_scheduling.py index cc6b5a713..7c5e4256d 100644 --- a/test/registered/scheduler/test_routing_key_scheduling.py +++ b/test/registered/scheduler/test_routing_key_scheduling.py @@ -20,7 +20,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large") +register_cuda_ci(est_time=50, stage="weekly", runner_config="1-gpu-large") register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True) register_cpu_ci(est_time=55, suite="base-c-test-cpu") diff --git a/test/registered/utils/test_model_file_verifier.py b/test/registered/utils/test_model_file_verifier.py deleted file mode 100644 index 848c4422d..000000000 --- a/test/registered/utils/test_model_file_verifier.py +++ /dev/null @@ -1,349 +0,0 @@ -import hashlib -import json -import os -import shutil -import subprocess -import sys -import tempfile -import unittest -import warnings -from contextlib import nullcontext -from io import StringIO - -import requests -from huggingface_hub import snapshot_download - -from sglang.srt.utils import kill_process_tree -from sglang.srt.utils.model_file_verifier import ( - IntegrityError, - compute_sha256, - generate_checksums, - verify, -) -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - popen_launch_server, -) - -# Note: AMD registration removed - test_model_file_verifier fails on AMD -register_cpu_ci(est_time=146, suite="base-c-test-cpu") - -MODEL_NAME = "Qwen/Qwen3-0.6B" - - -# ======== Base Test Classes ======== - - -class _FakeModelTestCase(unittest.TestCase): - - FAKE_FILES = { - "model.safetensors": b"fake safetensors content " * 100, - "config.json": b'{"model_type": "llama"}', - "tokenizer.json": b'{"version": "1.0"}', - } - - def setUp(self): - self.test_dir = tempfile.mkdtemp() - for filename, content in self.FAKE_FILES.items(): - _create_test_file(self.test_dir, filename, content) - - def tearDown(self): - shutil.rmtree(self.test_dir, ignore_errors=True) - - -class _RealModelTestCase(unittest.TestCase): - - @classmethod - def setUpClass(cls): - cls.original_model_path = snapshot_download(MODEL_NAME) - - def setUp(self): - self.test_dir = tempfile.mkdtemp() - shutil.copytree(self.original_model_path, self.test_dir, dirs_exist_ok=True) - - def tearDown(self): - shutil.rmtree(self.test_dir, ignore_errors=True) - - -# ======== Unit Tests ======== - - -class TestModelFileVerifier(_FakeModelTestCase): - - def test_detect_bit_rot(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - generate_checksums(source=self.test_dir, output_path=checksums_file) - - target_file = os.path.join(self.test_dir, "model.safetensors") - _flip_bit_in_file(target_file, byte_offset=50, bit_position=3) - - with self.assertRaises(IntegrityError) as ctx: - verify(model_path=self.test_dir, checksums_source=checksums_file) - - self.assertIn("model.safetensors", str(ctx.exception)) - self.assertIn("mismatch", str(ctx.exception).lower()) - - def test_detect_missing_file(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - generate_checksums(source=self.test_dir, output_path=checksums_file) - - os.remove(os.path.join(self.test_dir, "config.json")) - - with self.assertRaises(IntegrityError) as ctx: - verify(model_path=self.test_dir, checksums_source=checksums_file) - - self.assertIn("config.json", str(ctx.exception)) - - def test_compute_sha256(self): - test_file = os.path.join(self.test_dir, "test.bin") - content = b"hello world" - with open(test_file, "wb") as f: - f.write(content) - - result = compute_sha256(file_path=test_file) - expected = hashlib.sha256(content).hexdigest() - self.assertEqual(result, expected) - - def test_parallel_checksum_computation(self): - for i in range(10): - _create_test_file( - self.test_dir, f"shard_{i}.safetensors", f"content_{i}".encode() * 1000 - ) - - checksums_file = os.path.join(self.test_dir, "checksums.json") - result = generate_checksums( - source=self.test_dir, output_path=checksums_file, max_workers=4 - ) - - self.assertGreaterEqual(len(result.files), 10) - - def test_generated_json_snapshot(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - generate_checksums(source=self.test_dir, output_path=checksums_file) - - with open(checksums_file) as f: - data = json.load(f) - - expected = { - "files": { - "config.json": { - "sha256": "81dddc8c379baae137d99d24c5fa081d3a5ce52b6a221ddc22fe364711f8beaf", - "size": 23, - }, - "model.safetensors": { - "sha256": "eb0c73a48a89fefb6b68dd41af830d75610c885135eac99139373b04705d05f3", - "size": 2500, - }, - "tokenizer.json": { - "sha256": "4e3043229142b64d998563bc543ce034e0a2251af5d404995e3afcb8ce8850df", - "size": 18, - }, - } - } - self.assertEqual(data, expected) - - def test_legacy_checksums_format_deprecated(self): - legacy_data = { - "checksums": { - "model.safetensors": "eb0c73a48a89fefb6b68dd41af830d75610c885135eac99139373b04705d05f3", - "config.json": "81dddc8c379baae137d99d24c5fa081d3a5ce52b6a221ddc22fe364711f8beaf", - "tokenizer.json": "4e3043229142b64d998563bc543ce034e0a2251af5d404995e3afcb8ce8850df", - } - } - legacy_file = os.path.join(self.test_dir, "legacy_checksums.json") - with open(legacy_file, "w") as f: - json.dump(legacy_data, f) - - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - verify(model_path=self.test_dir, checksums_source=legacy_file) - self.assertEqual(len(w), 1) - self.assertTrue(issubclass(w[0].category, DeprecationWarning)) - self.assertIn("deprecated", str(w[0].message).lower()) - - -# ======== CLI Tests ======== - - -class TestModelFileVerifierCLI(_FakeModelTestCase): - - def test_cli_generate(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - result = subprocess.run( - [ - sys.executable, - "-m", - "sglang.srt.utils.model_file_verifier", - "generate", - "--model-path", - self.test_dir, - "--model-checksum", - checksums_file, - ], - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertTrue(os.path.exists(checksums_file)) - - with open(checksums_file) as f: - data = json.load(f) - self.assertIn("files", data) - self.assertEqual(len(data["files"]), 3) - - def test_cli_verify_success(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - generate_checksums(source=self.test_dir, output_path=checksums_file) - - result = subprocess.run( - [ - sys.executable, - "-m", - "sglang.srt.utils.model_file_verifier", - "verify", - "--model-path", - self.test_dir, - "--model-checksum", - checksums_file, - ], - capture_output=True, - text=True, - ) - self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}") - self.assertIn("verified successfully", result.stdout) - - def test_cli_verify_fails_on_corruption(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - generate_checksums(source=self.test_dir, output_path=checksums_file) - - target_file = os.path.join(self.test_dir, "model.safetensors") - _flip_bit_in_file(target_file, byte_offset=50, bit_position=3) - - result = subprocess.run( - [ - sys.executable, - "-m", - "sglang.srt.utils.model_file_verifier", - "verify", - "--model-path", - self.test_dir, - "--model-checksum", - checksums_file, - ], - capture_output=True, - text=True, - ) - self.assertNotEqual(result.returncode, 0) - combined = result.stdout + result.stderr - self.assertTrue( - "IntegrityError" in combined or "mismatch" in combined.lower(), - f"Expected integrity error, got: {combined}", - ) - - -# ======== HuggingFace Tests ======== - - -class TestModelFileVerifierHF(_RealModelTestCase): - - def test_generate_checksums_from_hf(self): - checksums_file = os.path.join(self.test_dir, "checksums.json") - result = generate_checksums(source=MODEL_NAME, output_path=checksums_file) - - self.assertTrue(os.path.exists(checksums_file)) - self.assertGreater(len(result.files), 0) - for filename, file_info in result.files.items(): - self.assertEqual(len(file_info.sha256), 64) - - def test_verify_with_hf_checksums_source(self): - verify(model_path=self.test_dir, checksums_source=MODEL_NAME) - - -# ======== Real Model E2E Tests ======== - - -class TestModelFileVerifierWithRealModel(_RealModelTestCase): - - def _run_server_test(self, *, corrupt_weights: bool, use_hf_checksum: bool): - if use_hf_checksum: - checksum_arg = MODEL_NAME - else: - checksums_file = os.path.join(self.test_dir, "checksums.json") - generate_checksums(source=self.test_dir, output_path=checksums_file) - checksum_arg = checksums_file - - corrupted_file = None - if corrupt_weights: - safetensors_files = [ - f for f in os.listdir(self.test_dir) if f.endswith(".safetensors") - ] - self.assertTrue(len(safetensors_files) > 0, "No safetensors files found") - corrupted_file = safetensors_files[0] - _flip_bit_in_file(os.path.join(self.test_dir, corrupted_file)) - - stdout_io, stderr_io = StringIO(), StringIO() - ctx = self.assertRaises(Exception) if corrupt_weights else nullcontext() - with ctx: - process = popen_launch_server( - model=self.test_dir, - base_url=DEFAULT_URL_FOR_TEST, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--model-checksum", checksum_arg], - return_stdout_stderr=(stdout_io, stderr_io), - ) - - if corrupt_weights: - output = stdout_io.getvalue() + stderr_io.getvalue() - self.assertIn(corrupted_file, output) - self.assertIn("mismatch", output.lower()) - else: - try: - response = requests.post( - f"{DEFAULT_URL_FOR_TEST}/generate", - json={"text": "Hello", "sampling_params": {"max_new_tokens": 8}}, - ) - self.assertEqual(response.status_code, 200) - self.assertIn("text", response.json()) - finally: - kill_process_tree(process.pid) - - def test_server_launch_with_checksum_intact(self): - self._run_server_test(corrupt_weights=False, use_hf_checksum=False) - - def test_server_launch_fails_with_corrupted_weights(self): - self._run_server_test(corrupt_weights=True, use_hf_checksum=False) - - def test_server_launch_with_hf_checksum_intact(self): - self._run_server_test(corrupt_weights=False, use_hf_checksum=True) - - def test_server_launch_with_hf_checksum_corrupted(self): - self._run_server_test(corrupt_weights=True, use_hf_checksum=True) - - -# ======== Test Utilities ======== - - -def _create_test_file(directory: str, filename: str, content: bytes) -> str: - path = os.path.join(directory, filename) - with open(path, "wb") as f: - f.write(content) - return path - - -def _flip_bit_in_file(file_path: str, byte_offset: int = 100, bit_position: int = 0): - file_size = os.path.getsize(file_path) - assert ( - byte_offset < file_size - ), f"byte_offset {byte_offset} >= file_size {file_size}" - - with open(file_path, "r+b") as f: - f.seek(byte_offset) - original_byte = f.read(1)[0] - f.seek(byte_offset) - f.write(bytes([original_byte ^ (1 << bit_position)])) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/run_suite.py b/test/run_suite.py index c60b6e03c..b7d46dfee 100644 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -188,12 +188,20 @@ NIGHTLY_SUITES = { OTHER_SUITES = { HWBackend.CPU: [ "default", + # `stage="weekly"`, dispatched by weekly-test-cpu.yml. + "weekly-test-cpu", ], HWBackend.CUDA: [ "stress", # `stage="weekly"` -- same shape. The three dicts group names for - # readability only; validation reads their union. + # readability only; validation reads their union. One entry per row of + # the matrix in weekly-test-nvidia.yml. + "weekly-test-1-gpu-large", + "weekly-test-2-gpu-large", + "weekly-test-4-gpu-h100", + "weekly-test-4-gpu-b200", "weekly-test-8-gpu-h200", + "weekly-test-8-gpu-b200", ], }