minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Move tests onto the right CI stages (#34074)

Browse Source 
Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
This commit is contained in:
Liangsheng Yin
2026-08-31 12:39:59 -07:00
committed by GitHub
co-authored by Baizhou Zhang
parent 579270d459
commit 95f0f41021
75 changed files with 196 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/8-gpu-models/test_glm_46.py
+2 -2
View File
@@ -7,8 +7,8 @@ from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
# Runs on both H200 and B200: registered once per runner_config below
register_cuda_ci(est_time=1320, stage="nightly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=1320, stage="nightly", runner_config="8-gpu-b200")
register_cuda_ci(est_time=1320, stage="weekly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=1320, stage="weekly", runner_config="8-gpu-b200")
GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
test/registered/8-gpu-models/test_minimax_m25.py
+2 -2
View File
@@ -7,8 +7,8 @@ from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
# Runs on both H200 and B200: registered once per runner_config below
register_cuda_ci(est_time=1860, stage="nightly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=1860, stage="nightly", runner_config="8-gpu-b200")
register_cuda_ci(est_time=1860, stage="weekly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=1860, stage="weekly", runner_config="8-gpu-b200")
MINIMAX_M25_MODEL_PATH = "MiniMaxAI/MiniMax-M2.5"
test/registered/8-gpu-models/test_mistral_large3.py
+2 -2
View File
@@ -10,8 +10,8 @@ from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
# Runs on both H200 and B200: registered once per runner_config below
# Note: trtllm_mla backend may have hardware-specific behavior
register_cuda_ci(est_time=3000, stage="nightly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=3000, stage="nightly", runner_config="8-gpu-b200")
register_cuda_ci(est_time=3000, stage="weekly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=3000, stage="weekly", runner_config="8-gpu-b200")
MISTRAL_LARGE3_FP8_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
MISTRAL_LARGE3_NVFP4_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4"
test/registered/8-gpu-models/test_ring_2_5_1t.py
+2 -2
View File
@@ -5,8 +5,8 @@ from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.run_combined_tests import run_combined_tests
from sglang.test.test_utils import ModelLaunchSettings
register_cuda_ci(est_time=1680, stage="nightly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=1680, stage="nightly", runner_config="8-gpu-b200")
register_cuda_ci(est_time=1680, stage="weekly", runner_config="8-gpu-h200")
register_cuda_ci(est_time=1680, stage="weekly", runner_config="8-gpu-b200")
RING_2_5_1T_MODEL_PATH = "inclusionAI/Ring-2.5-1T"
test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py
-5
View File
@@ -20,7 +20,6 @@ from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -80,9 +79,6 @@ def remove_failing_models(model_str):
return ",".join(filtered)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 = remove_failing_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
)
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2 = remove_failing_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2
)
@@ -206,7 +202,6 @@ class TestNightlyGsm8KEval(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model_groups = [
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
(parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
test/registered/attention/test_deepseek_v3_deterministic.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_deterministic_utils import (
TestDeterministicBase,
)
register_cuda_ci(est_time=240, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=240, stage="weekly", runner_config="1-gpu-large")
DEEPSEEK_MODEL = "lmsys/sglang-ci-dsv3-test"
test/registered/attention/test_qwen3_next_deterministic.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_deterministic_utils import (
TestDeterministicBase,
)
register_cuda_ci(est_time=200, stage="nightly", runner_config="4-gpu-h100")
register_cuda_ci(est_time=200, stage="weekly", runner_config="4-gpu-h100")
QWEN3_NEXT = "Qwen/Qwen3-Next-80B-A3B-Instruct"
test/registered/backends/test_deepseek_v3_fp4_cutedsl_moe.py
+1 -1
View File
@@ -17,7 +17,7 @@ from sglang.test.test_utils import (
write_github_step_summary,
)
register_cuda_ci(est_time=900, stage="nightly", runner_config="4-gpu-b200")
register_cuda_ci(est_time=900, stage="extra-b", runner_config="4-gpu-b200")
FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4"
SERVER_LAUNCH_TIMEOUT = 1000
test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=1770, stage="nightly", runner_config="4-gpu-b200")
register_cuda_ci(est_time=1770, stage="weekly", runner_config="4-gpu-b200")
class FlashinferTrtllmGenMoeBackendFP8Base:
test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding.py
+1 -1
View File
@@ -11,7 +11,7 @@ import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase
register_cuda_ci(est_time=130, stage="nightly", runner_config="4-gpu-h100")
register_cuda_ci(est_time=130, stage="weekly", runner_config="4-gpu-h100")
class TestPCGWithEAGLE3(PCGSpecBase, unittest.TestCase):
test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_dflash.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.test.test_utils import (
CustomTestCase,
)
register_cuda_ci(est_time=110, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=110, stage="weekly", runner_config="1-gpu-large")
class TestPCGWithDFlash(PCGSpecBase, CustomTestCase):
test/registered/cuda_graph/piecewise/test_pcg_with_speculative_decoding_extra.py
+1 -1
View File
@@ -8,7 +8,7 @@ import unittest
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.pcg_spec_fixture import PCGSpecBase
register_cuda_ci(est_time=450, stage="nightly", runner_config="4-gpu-h100")
register_cuda_ci(est_time=450, stage="weekly", runner_config="4-gpu-h100")
class TestPCGWithMTP(PCGSpecBase, unittest.TestCase):
test/registered/debug_utils/comparator/aligner/entrypoint/test_executor.py
+1 -2
View File
@@ -33,8 +33,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
class TestExecuteSubPlans:
test/registered/debug_utils/comparator/aligner/entrypoint/test_planner.py
+1 -2
View File
@@ -25,8 +25,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def _make_meta(
test/registered/debug_utils/comparator/aligner/reorderer/test_executor.py
+1 -2
View File
@@ -26,8 +26,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _zigzag_order(cp_size: int) -> list[int]:
test/registered/debug_utils/comparator/aligner/reorderer/test_planner.py
+1 -2
View File
@@ -26,8 +26,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
class TestComputeReordererPlans:
test/registered/debug_utils/comparator/aligner/test_axis_aligner.py
+1 -2
View File
@@ -17,8 +17,7 @@ from sglang.srt.debug_utils.comparator.log_sink import log_sink
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
class TestComputeAxisAlignerPlan:
test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_loader.py
+1 -1
View File
@@ -19,7 +19,7 @@ from sglang.srt.debug_utils.comparator.log_sink import LogSink
from sglang.srt.debug_utils.comparator.output_types import ErrorLog, InfoLog
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
_sglang_plugin = _SGLangPlugin()
_megatron_plugin = _MegatronPlugin()
test/registered/debug_utils/comparator/aligner/token_aligner/test_aux_plugins.py
+1 -2
View File
@@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.aligner.token_aligner.smart.types import
from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
_sglang_plugin = _SGLangPlugin()
_megatron_plugin = _MegatronPlugin()
test/registered/debug_utils/comparator/aligner/token_aligner/test_concat_steps.py
+1 -2
View File
@@ -10,8 +10,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import apply_dim_names
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
class TestExecuteConcat:
test/registered/debug_utils/comparator/aligner/token_aligner/test_executor.py
+1 -2
View File
@@ -29,8 +29,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def _named(tensor: torch.Tensor, names: list[str]) -> torch.Tensor:
test/registered/debug_utils/comparator/aligner/token_aligner/test_planner.py
+1 -2
View File
@@ -23,8 +23,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import TokenLayout
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
class TestBuildTokenIndexSGLangThd:
test/registered/debug_utils/comparator/aligner/token_aligner/test_thd_seq_lens_loader.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.srt.debug_utils.comparator.aligner.token_aligner.smart.aux_plugins i
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def _save_pt(
test/registered/debug_utils/comparator/aligner/unsharder/test_executor.py
+1 -2
View File
@@ -30,8 +30,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
from sglang.srt.debug_utils.comparator.output_types import ReplicatedCheckResult
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _name_tensors(
test/registered/debug_utils/comparator/aligner/unsharder/test_parallel_info.py
+1 -2
View File
@@ -9,8 +9,7 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import AxisInfo
from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
class TestNormalizeParallelInfo:
test/registered/debug_utils/comparator/aligner/unsharder/test_planner.py
+1 -2
View File
@@ -19,8 +19,7 @@ from sglang.srt.debug_utils.comparator.aligner.unsharder.types import (
from sglang.srt.debug_utils.comparator.dims_spec import ParallelAxis, parse_dims
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
class TestComputeUnsharderPlan:
test/registered/debug_utils/comparator/dims_spec/test_dim_parser.py
+1 -2
View File
@@ -12,8 +12,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=6, stage="weekly", runner_config="cpu")
class TestParseDim:
test/registered/debug_utils/comparator/dims_spec/test_dims_parser.py
+1 -2
View File
@@ -15,8 +15,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu")
class TestSingletonDimUtilFilterOut:
test/registered/debug_utils/comparator/dims_spec/test_tensor_naming.py
+1 -2
View File
@@ -14,8 +14,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=7, stage="weekly", runner_config="cpu")
class TestFindDimIndex:
test/registered/debug_utils/comparator/dims_spec/test_types.py
+1 -2
View File
@@ -9,8 +9,7 @@ from sglang.srt.debug_utils.comparator.dims_spec import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu")
class TestDimConstants:
test/registered/debug_utils/comparator/tensor_comparator/test_comparator.py
+1 -2
View File
@@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import DiffInfo
from sglang.srt.debug_utils.comparator.threshold_dsl import DiffThresholdRule
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=20, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=20, stage="weekly", runner_config="cpu")
class TestComputeTensorInfo:
test/registered/debug_utils/comparator/tensor_comparator/test_formatter.py
+1 -1
View File
@@ -63,7 +63,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
_DEFAULT_PERCENTILE_LINES: list[str] = [
" [blue]p1 [/] -1.8000 -1.8000 [dim]+0.00e+00[/]",
test/registered/debug_utils/comparator/tensor_comparator/test_types.py
+1 -2
View File
@@ -21,8 +21,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _make_stats(**overrides) -> TensorStats:
test/registered/debug_utils/comparator/test_bundle_comparator.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.srt.debug_utils.dump_loader import ValueWithMeta
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def _save_tensor(
test/registered/debug_utils/comparator/test_bundle_matcher.py
+1 -1
View File
@@ -13,7 +13,7 @@ from sglang.srt.debug_utils.comparator.bundle_matcher import (
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def _make_row(
test/registered/debug_utils/comparator/test_display.py
+1 -1
View File
@@ -21,7 +21,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _render_rich(renderable: object) -> str:
test/registered/debug_utils/comparator/test_dp_utils.py
+1 -1
View File
@@ -12,7 +12,7 @@ from sglang.srt.debug_utils.comparator.dp_utils import (
from sglang.srt.debug_utils.dump_loader import ValueWithMeta
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=15, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=15, stage="weekly", runner_config="cpu")
def _make_sglang_meta(
test/registered/debug_utils/comparator/test_dump_loader.py
+1 -1
View File
@@ -7,7 +7,7 @@ import torch
from sglang.srt.debug_utils.dump_loader import read_tokenizer_path
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _save_pt(
test/registered/debug_utils/comparator/test_e2e_demo.py
+1 -1
View File
@@ -26,7 +26,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="default", nightly=True)
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
_EXP_NAME = "demo_exp"
test/registered/debug_utils/comparator/test_entrypoint.py
+1 -1
View File
@@ -30,7 +30,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper, _RecomputeStatus
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
_FIXED_EXP_NAME = "my_exp_name"
test/registered/debug_utils/comparator/test_log_sink.py
+1 -2
View File
@@ -11,8 +11,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
from sglang.srt.debug_utils.comparator.report_sink import report_sink
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _make_error_log(**overrides) -> ErrorLog:
test/registered/debug_utils/comparator/test_manually_verify.py
+1 -1
View File
@@ -23,7 +23,7 @@ import torch
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=60, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=60, stage="weekly", runner_config="cpu")
_PUBLISH_DIR: Path = Path("/tmp/comparator_manual_verify")
_PNG_MAGIC: bytes = b"\x89PNG"
test/registered/debug_utils/comparator/test_meta_overrider.py
+1 -2
View File
@@ -16,8 +16,7 @@ from sglang.srt.debug_utils.comparator.meta_overrider import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
# ───────────────────── Unit: MetaOverrideRule ─────────────────────
test/registered/debug_utils/comparator/test_model_validation.py
+1 -2
View File
@@ -44,8 +44,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.types import (
from sglang.srt.debug_utils.comparator.utils import Pair, _check_equal_lengths
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
class TestCheckEqualLengths:
test/registered/debug_utils/comparator/test_output_types.py
+1 -1
View File
@@ -60,7 +60,7 @@ from sglang.srt.debug_utils.comparator.output_types import (
from sglang.srt.debug_utils.comparator.utils import Pair
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _render_rich(renderable: object) -> str:
test/registered/debug_utils/comparator/test_per_token_visualizer.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.srt.debug_utils.comparator.tensor_comparator.comparator import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
_PNG_MAGIC: bytes = b"\x89PNG"
test/registered/debug_utils/comparator/test_preset.py
+1 -2
View File
@@ -3,8 +3,7 @@ import pytest
from sglang.srt.debug_utils.comparator.preset import PRESETS, expand_preset
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=8, suite="base-c-test-cpu")
register_cpu_ci(est_time=8, stage="weekly", runner_config="cpu")
class TestExpandPreset:
test/registered/debug_utils/comparator/test_threshold_dsl.py
+1 -1
View File
@@ -11,7 +11,7 @@ from sglang.srt.debug_utils.comparator.threshold_dsl import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
def _ev(
test/registered/debug_utils/comparator/test_utils.py
+1 -2
View File
@@ -17,8 +17,7 @@ from sglang.srt.debug_utils.comparator.utils import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=7, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
class TestCalcRelDiff:
test/registered/debug_utils/comparator/test_visualizer.py
+1 -1
View File
@@ -10,7 +10,7 @@ from sglang.srt.debug_utils.comparator.visualizer.preprocessing import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
class TestPreprocessTensor:
test/registered/debug_utils/comparator/testing_helpers.py
+2 -2
View File
@@ -12,8 +12,8 @@ from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(
est_time=0,
suite="base-a-test-cpu",
nightly=True,
stage="weekly",
runner_config="cpu",
disabled="helper module, no tests",
)
test/registered/debug_utils/source_patcher/test_code_patcher.py
+1 -2
View File
@@ -10,8 +10,7 @@ from sglang.srt.debug_utils.source_patcher.code_patcher import (
from sglang.srt.debug_utils.source_patcher.types import EditSpec, PatchSpec
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=5, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
SAMPLE_MODULE_NAME = "_source_patcher_test_fixtures.sample_module"
test/registered/debug_utils/source_patcher/test_dumper_integration.py
+1 -2
View File
@@ -8,8 +8,7 @@ import yaml
from sglang.srt.debug_utils.dumper import DumperConfig, _Dumper
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=6, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
SAMPLE_MODULE_NAME = "_source_patcher_test_fixtures.sample_module"
test/registered/debug_utils/source_patcher/test_source_editor.py
+1 -2
View File
@@ -5,8 +5,7 @@ from sglang.srt.debug_utils.source_patcher.source_editor import apply_edits
from sglang.srt.debug_utils.source_patcher.types import EditSpec, PatchApplicationError
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=4, suite="base-c-test-cpu")
register_cpu_ci(est_time=10, stage="weekly", runner_config="cpu")
class TestApplyEdits:
test/registered/debug_utils/test_dump_comparator.py
+1 -1
View File
@@ -9,7 +9,7 @@ from sglang.srt.debug_utils.dump_comparator import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
# ----------------------------- Unit tests -----------------------------
test/registered/debug_utils/test_dump_loader.py
+1 -1
View File
@@ -15,7 +15,7 @@ from sglang.srt.debug_utils.dump_loader import (
)
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=30, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=30, stage="weekly", runner_config="cpu")
class TestReadMeta:
test/registered/debug_utils/test_schedule_simulator.py
+1 -2
View File
@@ -25,8 +25,7 @@ from sglang.srt.debug_utils.schedule_simulator import (
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=120, suite="base-a-test-cpu", nightly=True)
register_cpu_ci(est_time=357, suite="base-c-test-cpu")
register_cpu_ci(est_time=357, stage="weekly", runner_config="cpu")
# ==================== Non-E2E Tests ====================
test/registered/eval/test_text_models_gsm8k_eval.py
+2 -11
View File
@@ -9,7 +9,6 @@ from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
DEFAULT_URL_FOR_TEST,
ModelLaunchSettings,
@@ -23,21 +22,15 @@ from sglang.test.test_utils import (
# downloading on cache miss. Use a longer timeout than the default 600s.
NIGHTLY_EVAL_SERVER_TIMEOUT = 1800
register_cuda_ci(est_time=2880, stage="nightly", runner_config="2-gpu-large")
register_cuda_ci(est_time=2880, stage="weekly", runner_config="2-gpu-large")
MODEL_SCORE_THRESHOLDS = {
# sgl-eval (zero-shot chat, \boxed{}, math_verify grading). Thresholds are
# measured_score - 0.05, baselined on H100 2-GPU over the full 1319 split.
"meta-llama/Llama-3.1-8B-Instruct": 0.77, # 81.05% measured - 5%
"Qwen/Qwen3-8B": 0.76, # 81.43% measured - 5%
"Qwen/Qwen3-4B": 0.77, # 82.41% measured - 5%
"meta-llama/Llama-3.1-70B-Instruct": 0.90, # 94.77% measured - 5%
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.39, # 43.52% measured - 5%
"Qwen/Qwen2-57B-A14B-Instruct": 0.46, # 50.87% measured - 5%
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.77, # 82.34% measured - 5%
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.23, # 27.82% measured - 5%
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.80, # 84.91% measured - 5%
"zai-org/GLM-4.5-Air-FP8": 0.73, # 77.48% measured - 5%
"neuralmagic/gemma-2-2b-it-FP8": 0.02, # 6.52% measured - 5%
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.01% measured - 5%
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.35, # 40.33% measured - 5%
@@ -51,9 +44,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.models = []
models_tp1 = parse_models(
DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1
) + parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1)
models_tp1 = parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1)
for model_path in models_tp1:
cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
test/registered/kernels/benchmark/kv_canary/bench_plan.py
+1 -1
View File
@@ -25,7 +25,7 @@ from sglang.kernels.ops.kv_canary.verify import VerifyPlan
from sglang.kernels.ops.kv_canary.write import WritePlan
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=40, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=40, stage="weekly", runner_config="1-gpu-large")
# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
register_amd_ci(est_time=900, suite="nightly-amd-kernel-1-gpu", nightly=True)
test/registered/kernels/benchmark/kv_canary/bench_scatter_req_token_ids.py
+1 -1
View File
@@ -16,7 +16,7 @@ from sglang.kernels.ops.kv_canary.scatter_req_token_ids import (
)
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large")
# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
# (_BS_AXIS_CI/_SEQ_LEN_AXIS_CI via get_benchmark_range), same as CUDA nightly.
test/registered/kernels/benchmark/kv_canary/bench_verify.py
+1 -1
View File
@@ -32,7 +32,7 @@ from sglang.kernels.ops.kv_canary.verify import (
)
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large")
# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
# (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly.
test/registered/kernels/benchmark/kv_canary/bench_write.py
+1 -1
View File
@@ -30,7 +30,7 @@ from sglang.kernels.ops.kv_canary.verify import (
from sglang.kernels.ops.kv_canary.write import WritePlan, launch_canary_write_kernel
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
register_cuda_ci(est_time=20, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=20, stage="weekly", runner_config="1-gpu-large")
# AMD mirrors the CUDA nightly registration (nightly-only, no per-PR suite).
# Note: amd_ci_exec.sh sets SGLANG_IS_IN_CI, so this runs the CI-reduced range
# (build_fast_matrix_cases via get_benchmark_range), same as CUDA nightly.
test/registered/lora/test_lora_deepseek_v3_base_logprob_diff.py
+1 -1
View File
@@ -34,7 +34,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=1800, stage="nightly", runner_config="8-gpu-b200")
register_cuda_ci(est_time=1800, stage="weekly", runner_config="8-gpu-b200")
BASE_MODEL = "deepseek-ai/DeepSeek-V3.1-Base"
LORA_HF_REPO = "yushengsu/lora-diff-DeepSeek-V3.1-Base"
test/registered/lora/test_lora_kimi_k25_logprob_diff.py
+1 -1
View File
@@ -34,7 +34,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=420, stage="nightly", runner_config="8-gpu-b200")
register_cuda_ci(est_time=420, stage="weekly", runner_config="8-gpu-b200")
BASE_MODEL = "moonshotai/Kimi-K2.5"
LORA_HF_REPO = "yushengsu/lora-diff-Kimi-K2.5"
test/registered/lora/test_lora_openai_compatible.py
+1 -1
View File
@@ -14,7 +14,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=180, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=180, stage="weekly", runner_config="1-gpu-large")
def setup_class(cls, *, enable_lora):
test/registered/model_loading/test_prefetch_checkpoints_multi_gpu.py
+1 -1
View File
@@ -4,7 +4,7 @@ import sglang as sgl
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=300, stage="nightly", runner_config="4-gpu-h100")
register_cuda_ci(est_time=300, stage="weekly", runner_config="4-gpu-h100")
PROMPTS = [
"Hello, my name is",
test/registered/model_loading/test_runai_model_loader.py
+1 -1
View File
@@ -5,7 +5,7 @@ from sglang.srt.utils.common import temp_set_env
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=50, stage="weekly", runner_config="1-gpu-large")
TEST_GCS_MODEL = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
test/registered/scheduler/test_routing_key_scheduling.py
+1 -1
View File
@@ -20,7 +20,7 @@ from sglang.test.test_utils import (
popen_launch_server,
)
register_cuda_ci(est_time=50, stage="nightly", runner_config="1-gpu-large")
register_cuda_ci(est_time=50, stage="weekly", runner_config="1-gpu-large")
register_amd_ci(est_time=120, suite="nightly-amd-1-gpu", nightly=True)
register_cpu_ci(est_time=55, suite="base-c-test-cpu")
test/registered/utils/test_model_file_verifier.py
-349
View File
@@ -1,349 +0,0 @@
import hashlib
import json
import os
import shutil
import subprocess
import sys
import tempfile
import unittest
import warnings
from contextlib import nullcontext
from io import StringIO
import requests
from huggingface_hub import snapshot_download
from sglang.srt.utils import kill_process_tree
from sglang.srt.utils.model_file_verifier import (
IntegrityError,
compute_sha256,
generate_checksums,
verify,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
popen_launch_server,
)
# Note: AMD registration removed - test_model_file_verifier fails on AMD
register_cpu_ci(est_time=146, suite="base-c-test-cpu")
MODEL_NAME = "Qwen/Qwen3-0.6B"
# ======== Base Test Classes ========
class _FakeModelTestCase(unittest.TestCase):
FAKE_FILES = {
"model.safetensors": b"fake safetensors content " * 100,
"config.json": b'{"model_type": "llama"}',
"tokenizer.json": b'{"version": "1.0"}',
}
def setUp(self):
self.test_dir = tempfile.mkdtemp()
for filename, content in self.FAKE_FILES.items():
_create_test_file(self.test_dir, filename, content)
def tearDown(self):
shutil.rmtree(self.test_dir, ignore_errors=True)
class _RealModelTestCase(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.original_model_path = snapshot_download(MODEL_NAME)
def setUp(self):
self.test_dir = tempfile.mkdtemp()
shutil.copytree(self.original_model_path, self.test_dir, dirs_exist_ok=True)
def tearDown(self):
shutil.rmtree(self.test_dir, ignore_errors=True)
# ======== Unit Tests ========
class TestModelFileVerifier(_FakeModelTestCase):
def test_detect_bit_rot(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
generate_checksums(source=self.test_dir, output_path=checksums_file)
target_file = os.path.join(self.test_dir, "model.safetensors")
_flip_bit_in_file(target_file, byte_offset=50, bit_position=3)
with self.assertRaises(IntegrityError) as ctx:
verify(model_path=self.test_dir, checksums_source=checksums_file)
self.assertIn("model.safetensors", str(ctx.exception))
self.assertIn("mismatch", str(ctx.exception).lower())
def test_detect_missing_file(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
generate_checksums(source=self.test_dir, output_path=checksums_file)
os.remove(os.path.join(self.test_dir, "config.json"))
with self.assertRaises(IntegrityError) as ctx:
verify(model_path=self.test_dir, checksums_source=checksums_file)
self.assertIn("config.json", str(ctx.exception))
def test_compute_sha256(self):
test_file = os.path.join(self.test_dir, "test.bin")
content = b"hello world"
with open(test_file, "wb") as f:
f.write(content)
result = compute_sha256(file_path=test_file)
expected = hashlib.sha256(content).hexdigest()
self.assertEqual(result, expected)
def test_parallel_checksum_computation(self):
for i in range(10):
_create_test_file(
self.test_dir, f"shard_{i}.safetensors", f"content_{i}".encode() * 1000
)
checksums_file = os.path.join(self.test_dir, "checksums.json")
result = generate_checksums(
source=self.test_dir, output_path=checksums_file, max_workers=4
)
self.assertGreaterEqual(len(result.files), 10)
def test_generated_json_snapshot(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
generate_checksums(source=self.test_dir, output_path=checksums_file)
with open(checksums_file) as f:
data = json.load(f)
expected = {
"files": {
"config.json": {
"sha256": "81dddc8c379baae137d99d24c5fa081d3a5ce52b6a221ddc22fe364711f8beaf",
"size": 23,
},
"model.safetensors": {
"sha256": "eb0c73a48a89fefb6b68dd41af830d75610c885135eac99139373b04705d05f3",
"size": 2500,
},
"tokenizer.json": {
"sha256": "4e3043229142b64d998563bc543ce034e0a2251af5d404995e3afcb8ce8850df",
"size": 18,
},
}
}
self.assertEqual(data, expected)
def test_legacy_checksums_format_deprecated(self):
legacy_data = {
"checksums": {
"model.safetensors": "eb0c73a48a89fefb6b68dd41af830d75610c885135eac99139373b04705d05f3",
"config.json": "81dddc8c379baae137d99d24c5fa081d3a5ce52b6a221ddc22fe364711f8beaf",
"tokenizer.json": "4e3043229142b64d998563bc543ce034e0a2251af5d404995e3afcb8ce8850df",
}
}
legacy_file = os.path.join(self.test_dir, "legacy_checksums.json")
with open(legacy_file, "w") as f:
json.dump(legacy_data, f)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
verify(model_path=self.test_dir, checksums_source=legacy_file)
self.assertEqual(len(w), 1)
self.assertTrue(issubclass(w[0].category, DeprecationWarning))
self.assertIn("deprecated", str(w[0].message).lower())
# ======== CLI Tests ========
class TestModelFileVerifierCLI(_FakeModelTestCase):
def test_cli_generate(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
result = subprocess.run(
[
sys.executable,
"-m",
"sglang.srt.utils.model_file_verifier",
"generate",
"--model-path",
self.test_dir,
"--model-checksum",
checksums_file,
],
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
self.assertTrue(os.path.exists(checksums_file))
with open(checksums_file) as f:
data = json.load(f)
self.assertIn("files", data)
self.assertEqual(len(data["files"]), 3)
def test_cli_verify_success(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
generate_checksums(source=self.test_dir, output_path=checksums_file)
result = subprocess.run(
[
sys.executable,
"-m",
"sglang.srt.utils.model_file_verifier",
"verify",
"--model-path",
self.test_dir,
"--model-checksum",
checksums_file,
],
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, f"stderr: {result.stderr}")
self.assertIn("verified successfully", result.stdout)
def test_cli_verify_fails_on_corruption(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
generate_checksums(source=self.test_dir, output_path=checksums_file)
target_file = os.path.join(self.test_dir, "model.safetensors")
_flip_bit_in_file(target_file, byte_offset=50, bit_position=3)
result = subprocess.run(
[
sys.executable,
"-m",
"sglang.srt.utils.model_file_verifier",
"verify",
"--model-path",
self.test_dir,
"--model-checksum",
checksums_file,
],
capture_output=True,
text=True,
)
self.assertNotEqual(result.returncode, 0)
combined = result.stdout + result.stderr
self.assertTrue(
"IntegrityError" in combined or "mismatch" in combined.lower(),
f"Expected integrity error, got: {combined}",
)
# ======== HuggingFace Tests ========
class TestModelFileVerifierHF(_RealModelTestCase):
def test_generate_checksums_from_hf(self):
checksums_file = os.path.join(self.test_dir, "checksums.json")
result = generate_checksums(source=MODEL_NAME, output_path=checksums_file)
self.assertTrue(os.path.exists(checksums_file))
self.assertGreater(len(result.files), 0)
for filename, file_info in result.files.items():
self.assertEqual(len(file_info.sha256), 64)
def test_verify_with_hf_checksums_source(self):
verify(model_path=self.test_dir, checksums_source=MODEL_NAME)
# ======== Real Model E2E Tests ========
class TestModelFileVerifierWithRealModel(_RealModelTestCase):
def _run_server_test(self, *, corrupt_weights: bool, use_hf_checksum: bool):
if use_hf_checksum:
checksum_arg = MODEL_NAME
else:
checksums_file = os.path.join(self.test_dir, "checksums.json")
generate_checksums(source=self.test_dir, output_path=checksums_file)
checksum_arg = checksums_file
corrupted_file = None
if corrupt_weights:
safetensors_files = [
f for f in os.listdir(self.test_dir) if f.endswith(".safetensors")
]
self.assertTrue(len(safetensors_files) > 0, "No safetensors files found")
corrupted_file = safetensors_files[0]
_flip_bit_in_file(os.path.join(self.test_dir, corrupted_file))
stdout_io, stderr_io = StringIO(), StringIO()
ctx = self.assertRaises(Exception) if corrupt_weights else nullcontext()
with ctx:
process = popen_launch_server(
model=self.test_dir,
base_url=DEFAULT_URL_FOR_TEST,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=["--model-checksum", checksum_arg],
return_stdout_stderr=(stdout_io, stderr_io),
)
if corrupt_weights:
output = stdout_io.getvalue() + stderr_io.getvalue()
self.assertIn(corrupted_file, output)
self.assertIn("mismatch", output.lower())
else:
try:
response = requests.post(
f"{DEFAULT_URL_FOR_TEST}/generate",
json={"text": "Hello", "sampling_params": {"max_new_tokens": 8}},
)
self.assertEqual(response.status_code, 200)
self.assertIn("text", response.json())
finally:
kill_process_tree(process.pid)
def test_server_launch_with_checksum_intact(self):
self._run_server_test(corrupt_weights=False, use_hf_checksum=False)
def test_server_launch_fails_with_corrupted_weights(self):
self._run_server_test(corrupt_weights=True, use_hf_checksum=False)
def test_server_launch_with_hf_checksum_intact(self):
self._run_server_test(corrupt_weights=False, use_hf_checksum=True)
def test_server_launch_with_hf_checksum_corrupted(self):
self._run_server_test(corrupt_weights=True, use_hf_checksum=True)
# ======== Test Utilities ========
def _create_test_file(directory: str, filename: str, content: bytes) -> str:
path = os.path.join(directory, filename)
with open(path, "wb") as f:
f.write(content)
return path
def _flip_bit_in_file(file_path: str, byte_offset: int = 100, bit_position: int = 0):
file_size = os.path.getsize(file_path)
assert (
byte_offset < file_size
), f"byte_offset {byte_offset} >= file_size {file_size}"
with open(file_path, "r+b") as f:
f.seek(byte_offset)
original_byte = f.read(1)[0]
f.seek(byte_offset)
f.write(bytes([original_byte ^ (1 << bit_position)]))
if __name__ == "__main__":
unittest.main()
test/run_suite.py
+9 -1
View File
@@ -188,12 +188,20 @@ NIGHTLY_SUITES = {
OTHER_SUITES = {
HWBackend.CPU: [
"default",
# `stage="weekly"`, dispatched by weekly-test-cpu.yml.
"weekly-test-cpu",
],
HWBackend.CUDA: [
"stress",
# `stage="weekly"` -- same shape. The three dicts group names for
# readability only; validation reads their union.
# readability only; validation reads their union. One entry per row of
# the matrix in weekly-test-nvidia.yml.
"weekly-test-1-gpu-large",
"weekly-test-2-gpu-large",
"weekly-test-4-gpu-h100",
"weekly-test-4-gpu-b200",
"weekly-test-8-gpu-h200",
"weekly-test-8-gpu-b200",
],
}