feature: revamp nightly tests with combined runner (#15324)

2025-12-20 19:25:22 -08:00
parent 9a3bdf2c95
commit 96740d6983
23 changed files with 1332 additions and 1420 deletions
@@ -0,0 +1,79 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+DEEPSEEK_V31_MODEL_PATH = "deepseek-ai/DeepSeek-V3.1"
+
+
+@unittest.skipIf(not is_blackwell_system(), "Requires B200")
+class TestDeepseekV31Unified(unittest.TestCase):
+    """Unified test class for DeepSeek-V3.1 performance and accuracy.
+
+    Two variants:
+    - basic: Standard TP=8
+    - mtp: TP=8 + EAGLE speculative decoding
+
+    Each variant runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_deepseek_v31_all_variants(self):
+        """Run performance and accuracy for all DeepSeek-V3.1 variants."""
+        # Define base arguments shared by most variants
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true}',
+        ]
+        mtp_args = [
+            "--speculative-algorithm=EAGLE",
+            "--speculative-num-steps=3",
+            "--speculative-eagle-topk=1",
+            "--speculative-num-draft-tokens=4",
+            "--mem-frac=0.7",
+        ]
+
+        variants = [
+            # Variant: "basic" - Standard TP=8
+            ModelLaunchSettings(
+                DEEPSEEK_V31_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+            # Variant: "mtp" - TP=8 + EAGLE speculative decoding
+            ModelLaunchSettings(
+                DEEPSEEK_V31_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args + mtp_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.1 Unified",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=0.935
+            ),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_deepseek_v31",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,194 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+register_cuda_ci(est_time=8000, suite="nightly-8-gpu-common", nightly=True)
+
+DEEPSEEK_V32_MODEL_PATH = "deepseek-ai/DeepSeek-V3.2"
+
+BASE_ARGS = [
+    "--trust-remote-code",
+    "--model-loader-extra-config",
+    '{"enable_multithread_load": true}',
+]
+
+DP_ARGS = [
+    "--tp=8",
+    "--dp=8",
+    "--enable-dp-attention",
+]
+
+# Accuracy thresholds
+GSM8K_BASELINE = 0.935
+GPQA_BASELINE = 0.835
+
+
+class TestDeepseekV32Unified(unittest.TestCase):
+    """Unified test class for DeepSeek V3.2 performance and accuracy.
+
+    Tests multiple variants with both performance and accuracy tests:
+    - dp: Standard TP=8 + DP=8 with dp-attention
+    - dp+mtp: DP + EAGLE speculative decoding
+    - tp: Pure TP=8 only
+    - tp+mtp: Pure TP=8 + EAGLE speculative decoding
+    """
+
+    @unittest.skipIf(is_blackwell_system(), "Requires H200 system")
+    def test_deepseek_v32_all_variants(self):
+        """Run performance and accuracy for all DeepSeek V3.2 variants."""
+        TP_ARGS = [
+            "--tp=8",
+        ]
+        MTP_ARGS = [
+            "--speculative-algorithm=EAGLE",
+            "--speculative-num-steps=3",
+            "--speculative-eagle-topk=1",
+            "--speculative-num-draft-tokens=4",
+            "--mem-frac=0.7",
+        ]
+        variants = [
+            # Variant: "dp" - Standard TP=8 + DP=8 with dp-attention
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS,
+            ),
+            # Variant: "dp+mtp" - DP + EAGLE speculative decoding
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + MTP_ARGS,
+            ),
+            # Variant: "tp" - Pure TP=8 only
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + TP_ARGS,
+            ),
+            # Variant: "tp+mtp" - Pure TP=8 + EAGLE speculative decoding
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + TP_ARGS + MTP_ARGS,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.2 Unified",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
+            ),
+            performance_params=PerformanceTestParams(
+                batch_sizes=[1, 8, 16, 64],
+                profile_dir="performance_profiles_deepseek_v32",
+            ),
+        )
+
+    @unittest.skipIf(is_blackwell_system(), "Requires H200 system")
+    def test_deepseek_v32_nsa_backends(self):
+        """Test NSA attention backend variants (H200 only).
+
+        Tests three NSA backend configurations:
+        - flashmla: flashmla_sparse prefill + flashmla_kv decode
+        - fa3: FA3 prefill + FA3 decode
+        - fp8kvcache: default backends with FP8 KV cache
+        """
+        NSA_FLASHMLA_ARGS = [
+            "--attention-backend=nsa",
+            "--nsa-prefill-backend=flashmla_sparse",
+            "--nsa-decode-backend=flashmla_kv",
+        ]
+
+        NSA_FA3_ARGS = [
+            "--attention-backend=nsa",
+            "--nsa-prefill-backend=fa3",
+            "--nsa-decode-backend=fa3",
+        ]
+
+        NSA_FP8KV_ARGS = [
+            "--attention-backend=nsa",
+            "--kv-cache-dtype=fp8_e4m3",
+        ]
+
+        nsa_variants = [
+            # flashmla backend
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + NSA_FLASHMLA_ARGS,
+            ),
+            # fa3 backend
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + NSA_FA3_ARGS,
+            ),
+            # fp8 kv cache
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + NSA_FP8KV_ARGS,
+            ),
+        ]
+
+        run_combined_tests(
+            models=nsa_variants,
+            test_name="DeepSeek-V3.2 NSA Backends",
+            accuracy_params=AccuracyTestParams(
+                dataset="gsm8k", baseline_accuracy=GSM8K_BASELINE
+            ),
+            performance_params=PerformanceTestParams(
+                batch_sizes=[1, 8, 16, 64],
+                profile_dir="performance_profiles_deepseek_v32_nsa",
+            ),
+        )
+
+    @unittest.skipIf(not is_blackwell_system(), "Requires B200")
+    def test_deepseek_v32_b200(self):
+        """Test DeepSeek V3.2 with GPQA evaluation using thinking mode (B200 only).
+
+        This test runs GPQA evaluation with the reasoning parser enabled.
+        """
+        B200_REASONING_ARGS = [
+            "--tool-call-parser=deepseekv32",
+            "--reasoning-parser=deepseek-v3",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                DEEPSEEK_V32_MODEL_PATH,
+                tp_size=8,
+                extra_args=BASE_ARGS + DP_ARGS + B200_REASONING_ARGS,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="DeepSeek-V3.2 GPQA (B200)",
+            accuracy_params=AccuracyTestParams(
+                dataset="gpqa",
+                baseline_accuracy=GPQA_BASELINE,
+                num_examples=198,
+                num_threads=198,
+                max_tokens=120000,
+                thinking_mode="deepseek-v3",
+                temperature=0.1,
+                repeat=4,
+            ),
+            performance_params=None,  # Skip performance test for GPQA
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,57 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+GLM_4_6_MODEL_PATH = "zai-org/GLM-4.6"
+
+
+class TestGLM46Unified(unittest.TestCase):
+    """Unified test class for GLM-4.6 performance and accuracy.
+
+    Single variant with simple TP=8 configuration.
+    GLM-4.6 is a 357B MoE model.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_glm_46(self):
+        """Run performance and accuracy for GLM-4.6."""
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                GLM_4_6_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="GLM-4.6 Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_glm_4_6",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,58 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+KIMI_K2_THINKING_MODEL_PATH = "moonshotai/Kimi-K2-Thinking"
+
+
+class TestKimiK2Unified(unittest.TestCase):
+    """Unified test class for Kimi-K2-Thinking performance and accuracy.
+
+    Single variant with TP=8 + tool/reasoning parsers.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner with extra_bench_args)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_kimi_k2(self):
+        """Run performance and accuracy for Kimi-K2-Thinking."""
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+            "--tool-call-parser=kimi_k2",
+            "--reasoning-parser=kimi_k2",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                KIMI_K2_THINKING_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Kimi-K2-Thinking Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.95),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_kimi_k2_thinking",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,60 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+MINIMAX_M2_MODEL_PATH = "MiniMaxAI/MiniMax-M2"
+
+
+class TestMiniMaxM2Unified(unittest.TestCase):
+    """Unified test class for MiniMax-M2 performance and accuracy.
+
+    Single variant with TP=8 + EP=8 configuration.
+    MiniMax-M2 is a 230B MoE model with 10B active params.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner with extra_bench_args)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_minimax_m2(self):
+        """Run performance and accuracy for MiniMax-M2."""
+        base_args = [
+            "--tp=8",
+            "--ep=8",
+            "--trust-remote-code",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true}',
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                MINIMAX_M2_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="MiniMax-M2 Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.80),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_minimax_m2",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,92 @@
+import os
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+# Note: trtllm_mla backend may have hardware-specific behavior
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+MISTRAL_LARGE3_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512"
+MISTRAL_LARGE3_EAGLE_MODEL_PATH = "mistralai/Mistral-Large-3-675B-Instruct-2512-Eagle"
+
+
+@unittest.skipIf(not is_blackwell_system(), "Requires B200")
+class TestMistralLarge3Unified(unittest.TestCase):
+    """Unified test class for Mistral-Large-3 performance and accuracy.
+
+    Two variants:
+    - basic: TP=8 + trtllm_mla backend
+    - eagle: basic + EAGLE speculative decoding with draft model
+
+    Each variant runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        # Set environment variable to disable JIT DeepGemm
+        os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"] = "0"
+
+    @classmethod
+    def tearDownClass(cls):
+        # Clean up environment variable
+        if "SGLANG_ENABLE_JIT_DEEPGEMM" in os.environ:
+            del os.environ["SGLANG_ENABLE_JIT_DEEPGEMM"]
+
+    def test_mistral_large3_all_variants(self):
+        """Run performance and accuracy for all Mistral-Large-3 variants."""
+        base_args = [
+            "--tp=8",
+            "--attention-backend=trtllm_mla",
+            "--model-loader-extra-config",
+            '{"enable_multithread_load": true}',
+            "--chat-template=mistral",
+        ]
+        eagle_args = [
+            "--speculative-algorithm=EAGLE",
+            f"--speculative-draft-model-path={MISTRAL_LARGE3_EAGLE_MODEL_PATH}",
+            "--speculative-num-steps=3",
+            "--speculative-eagle-topk=1",
+            "--speculative-num-draft-tokens=4",
+            "--kv-cache-dtype=auto",
+        ]
+
+        variants = [
+            # Variant: "basic" - TP=8 + trtllm_mla backend
+            ModelLaunchSettings(
+                MISTRAL_LARGE3_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+            # Variant: "eagle" - TP=8 + trtllm_mla + EAGLE with draft model
+            ModelLaunchSettings(
+                MISTRAL_LARGE3_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args + eagle_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Mistral-Large-3 Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.90),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_mistral_large3",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -0,0 +1,57 @@
+import sys
+import unittest
+from pathlib import Path
+
+# Add nightly directory to path for run_combined_tests import
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "nightly"))
+
+from accuracy_test_runner import AccuracyTestParams
+from performance_test_runner import PerformanceTestParams
+from run_combined_tests import run_combined_tests
+
+from sglang.test.ci.ci_register import register_cuda_ci
+from sglang.test.test_utils import ModelLaunchSettings, is_blackwell_system
+
+# Runs on both H200 and B200 via nightly-8-gpu-common suite
+register_cuda_ci(est_time=12000, suite="nightly-8-gpu-common", nightly=True)
+
+QWEN3_235B_MODEL_PATH = "Qwen/Qwen3-235B-A22B-Instruct-2507"
+
+
+@unittest.skipIf(not is_blackwell_system(), "Requires B200")
+class TestQwen3235BUnified(unittest.TestCase):
+    """Unified test class for Qwen3-235B performance and accuracy.
+
+    Single variant with simple TP=8 configuration.
+    Runs BOTH:
+    - Performance test (using NightlyBenchmarkRunner)
+    - Accuracy test (using run_eval with mgsm_en)
+    """
+
+    def test_qwen3_235b(self):
+        """Run performance and accuracy for Qwen3-235B."""
+        base_args = [
+            "--tp=8",
+            "--trust-remote-code",
+        ]
+
+        variants = [
+            ModelLaunchSettings(
+                QWEN3_235B_MODEL_PATH,
+                tp_size=8,
+                extra_args=base_args,
+            ),
+        ]
+
+        run_combined_tests(
+            models=variants,
+            test_name="Qwen3-235B Unified",
+            accuracy_params=AccuracyTestParams(dataset="gsm8k", baseline_accuracy=0.88),
+            performance_params=PerformanceTestParams(
+                profile_dir="performance_profiles_qwen3_235b",
+            ),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()