[CI] Migrate nightly tests to test/registered/ (#15582)

test/registered/perf/test_gpt_oss_4gpu_perf.py

@@ -0,0 +1,60 @@
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.nightly_utils import NightlyBenchmarkRunner
+from sglang.test.test_utils import DEFAULT_URL_FOR_TEST
+
+register_cuda_ci(est_time=600, suite="nightly-4-gpu-b200", nightly=True)
+
+PROFILE_DIR = "performance_profiles_gpt_oss_4gpu"
+
+
+class TestNightlyGptOss4GpuPerformance(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.models = [
+            (
+                "openai/gpt-oss-120b",
+                [
+                    "--tp",
+                    "4",
+                    "--cuda-graph-max-bs",
+                    "200",
+                    "--mem-fraction-static",
+                    "0.93",
+                ],
+            ),
+        ]
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.batch_sizes = [1, 1, 8, 16, 64]
+        cls.input_lens = (4096,)
+        cls.output_lens = (512,)
+        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
+        cls.runner.setup_profile_directory()
+
+    def test_bench_one_batch(self):
+        all_model_succeed = True
+
+        for model_path, other_args in self.models:
+            with self.subTest(model=model_path):
+                results, success, _ = self.runner.run_benchmark_for_model(
+                    model_path=model_path,
+                    batch_sizes=self.batch_sizes,
+                    input_lens=self.input_lens,
+                    output_lens=self.output_lens,
+                    other_args=other_args,
+                )
+
+                if not success:
+                    all_model_succeed = False
+
+                self.runner.add_report(results)
+
+        self.runner.write_final_report()
+
+        if not all_model_succeed:
+            raise AssertionError("Some models failed the perf tests.")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/perf/test_text_models_perf.py

@@ -0,0 +1,62 @@
+import unittest
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.nightly_utils import NightlyBenchmarkRunner
+from sglang.test.test_utils import (
+    DEFAULT_URL_FOR_TEST,
+    ModelLaunchSettings,
+    _parse_int_list_env,
+    parse_models,
+)
+
+register_cuda_ci(est_time=3600, suite="nightly-perf-text-2-gpu", nightly=True)
+
+PROFILE_DIR = "performance_profiles_text_models"
+
+
+class TestNightlyTextModelsPerformance(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.models = []
+        # TODO: replace with DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1 or other model lists
+        for model_path in parse_models("meta-llama/Llama-3.1-8B-Instruct"):
+            cls.models.append(ModelLaunchSettings(model_path, tp_size=1))
+        for model_path in parse_models("Qwen/Qwen2-57B-A14B-Instruct"):
+            cls.models.append(ModelLaunchSettings(model_path, tp_size=2))
+        # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
+        # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
+        # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
+        # (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
+        cls.base_url = DEFAULT_URL_FOR_TEST
+        cls.batch_sizes = [1, 1, 8, 16, 64]
+        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_INPUT_LENS", "4096"))
+        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_OUTPUT_LENS", "512"))
+        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
+        cls.runner.setup_profile_directory()
+
+    def test_bench_one_batch(self):
+        all_model_succeed = True
+
+        for model_setup in self.models:
+            with self.subTest(model=model_setup.model_path):
+                results, success, _ = self.runner.run_benchmark_for_model(
+                    model_path=model_setup.model_path,
+                    batch_sizes=self.batch_sizes,
+                    input_lens=self.input_lens,
+                    output_lens=self.output_lens,
+                    other_args=model_setup.extra_args,
+                )
+
+                if not success:
+                    all_model_succeed = False
+
+                self.runner.add_report(results)
+
+        self.runner.write_final_report()
+
+        if not all_model_succeed:
+            raise AssertionError("Some models failed the perf tests.")
+
+
+if __name__ == "__main__":
+    unittest.main()

test/registered/perf/test_vlms_perf.py

@@ -0,0 +1,90 @@
+import os
+import unittest
+import warnings
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.nightly_utils import NightlyBenchmarkRunner
+from sglang.test.test_utils import (
+    DEFAULT_URL_FOR_TEST,
+    ModelLaunchSettings,
+    _parse_int_list_env,
+    parse_models,
+)
+
+register_cuda_ci(est_time=7200, suite="nightly-perf-vlm-2-gpu", nightly=True)
+
+PROFILE_DIR = "performance_profiles_vlms"
+
+MODEL_DEFAULTS = [
+    # Keep conservative defaults. Can be overridden by env NIGHTLY_VLM_MODELS
+    ModelLaunchSettings(
+        "Qwen/Qwen2.5-VL-7B-Instruct",
+        extra_args=["--mem-fraction-static=0.7"],
+    ),
+    ModelLaunchSettings(
+        "google/gemma-3-27b-it",
+    ),
+    ModelLaunchSettings("Qwen/Qwen3-VL-30B-A3B-Instruct", extra_args=["--tp=2"]),
+    # "OpenGVLab/InternVL2_5-2B",
+    # buggy in official transformers impl
+    # "openbmb/MiniCPM-V-2_6",
+]
+
+
+class TestNightlyVLMModelsPerformance(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        warnings.filterwarnings(
+            "ignore", category=ResourceWarning, message="unclosed.*socket"
+        )
+
+        nightly_vlm_models_str = os.environ.get("NIGHTLY_VLM_MODELS")
+        if nightly_vlm_models_str:
+            cls.models = []
+            model_paths = parse_models(nightly_vlm_models_str)
+            for model_path in model_paths:
+                cls.models.append(ModelLaunchSettings(model_path))
+        else:
+            cls.models = MODEL_DEFAULTS
+
+        cls.base_url = DEFAULT_URL_FOR_TEST
+
+        cls.batch_sizes = _parse_int_list_env("NIGHTLY_VLM_BATCH_SIZES", "1,1,2,8,16")
+        cls.input_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_INPUT_LENS", "4096"))
+        cls.output_lens = tuple(_parse_int_list_env("NIGHTLY_VLM_OUTPUT_LENS", "512"))
+        cls.runner = NightlyBenchmarkRunner(PROFILE_DIR, cls.__name__, cls.base_url)
+        cls.runner.setup_profile_directory()
+
+    def test_bench_one_batch(self):
+        all_model_succeed = True
+
+        for model_setup in self.models:
+            with self.subTest(model=model_setup.model_path):
+                # VLMs need additional benchmark args for dataset and trust-remote-code
+                extra_bench_args = [
+                    "--trust-remote-code",
+                    "--dataset-name=mmmu",
+                ]
+
+                results, success, _ = self.runner.run_benchmark_for_model(
+                    model_path=model_setup.model_path,
+                    batch_sizes=self.batch_sizes,
+                    input_lens=self.input_lens,
+                    output_lens=self.output_lens,
+                    other_args=model_setup.extra_args,
+                    extra_bench_args=extra_bench_args,
+                )
+
+                if not success:
+                    all_model_succeed = False
+
+                self.runner.add_report(results)
+
+        self.runner.write_final_report()
+
+        if not all_model_succeed:
+            raise AssertionError("Some models failed the perf tests.")
+
+
+if __name__ == "__main__":
+    unittest.main()