diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_triton.py b/test/manual/4-gpu-models/test_qwen35_fp4_triton.py similarity index 100% rename from test/registered/4-gpu-models/test_qwen35_fp4_triton.py rename to test/manual/4-gpu-models/test_qwen35_fp4_triton.py diff --git a/test/manual/4-gpu-models/test_qwen35_models_archived.py b/test/manual/4-gpu-models/test_qwen35_models_archived.py new file mode 100644 index 000000000..f0443c482 --- /dev/null +++ b/test/manual/4-gpu-models/test_qwen35_models_archived.py @@ -0,0 +1,168 @@ +"""Archived test classes split out of test/registered/4-gpu-models/test_qwen35_models.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/4-gpu-models/test_qwen35_models_archived.py`. +""" + +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.accuracy_test_runner import AccuracyTestParams +from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin + +# This eval harness applies the chat_template, which is critical for qwen3.5 +# to get good accuracy on gsm8k +from sglang.test.run_combined_tests import run_combined_tests +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + ModelLaunchSettings, + popen_launch_server, +) + +QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" +ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} + + +class TestQwen35FP4(CustomTestCase): + def test_gsm8k(self): + base_args = [ + "--tp-size", + "4", + "--chunked-prefill-size", + "2048", + "--mamba-scheduler-strategy", + "extra_buffer", + "--mamba-track-interval", + "128", + "--mamba-ssm-dtype", + "bfloat16", + "--max-running-requests", + "128", + "--reasoning-parser", + "qwen3", + "--attention-backend", + "trtllm_mha", + "--quantization", + "modelopt_fp4", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ] + + variants = [ + ModelLaunchSettings( + QWEN35_FP4_MODEL, + extra_args=base_args, + variant="Triton", + ), + # TODO: Fix this and re-enable it + # ModelLaunchSettings( + # QWEN35_FP4_MODEL, + # extra_args=base_args + ["--linear-attn-decode-backend", "flashinfer"], + # variant="FlashInfer", + # ), + ] + + run_combined_tests( + models=variants, + test_name="Qwen3.5-397B-A17B-NVFP4", + accuracy_params=AccuracyTestParams( + dataset="gsm8k", + baseline_accuracy=ACC_THRESHOLDS[QWEN35_FP4_MODEL]["gsm8k"], + num_examples=200, + num_threads=128, + max_tokens=16000, + thinking_mode="qwen3", + temperature=0.6, + top_p=0.95, + top_k=20, + ), + ) + + +class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): + reasoning_parser_name = "qwen3" + + @classmethod + def setUpClass(cls): + cls.model = QWEN35_FP4_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.init_reasoning_token_verifier() + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--chunked-prefill-size", + "2048", + "--mamba-scheduler-strategy", + "extra_buffer", + "--mamba-track-interval", + "128", + "--mamba-ssm-dtype", + "bfloat16", + "--max-running-requests", + "128", + "--reasoning-parser", + "qwen3", + "--attention-backend", + "trtllm_mha", + "--quantization", + "modelopt_fp4", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.8", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + model=self.model, + eval_name="gsm8k", + num_shots=5, + num_examples=200, + max_tokens=16000, + num_threads=128, + repeat=1, + temperature=0.6, + top_p=0.95, + top_k=20, + base_url=self.base_url, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"]) + + server_info = requests.get(self.base_url + "/server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print(f"{avg_spec_accept_length=}") + self.assertGreater(avg_spec_accept_length, 3.3) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/4-gpu-models/test_qwen3_next_models.py b/test/manual/4-gpu-models/test_qwen3_next_models.py similarity index 100% rename from test/registered/4-gpu-models/test_qwen3_next_models.py rename to test/manual/4-gpu-models/test_qwen3_next_models.py diff --git a/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py b/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py new file mode 100644 index 000000000..3f6142aee --- /dev/null +++ b/test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py @@ -0,0 +1,44 @@ +"""Archived test classes split out of test/registered/4-gpu-models/test_qwen3_next_models_mtp.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/4-gpu-models/test_qwen3_next_models_mtp_archived.py`. +""" + +import unittest + +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase + +QWEN3_NEXT_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" + + +class TestQwen3NextMTP(GSM8KMixin, KLDivergenceMixin, DefaultServerBase): + model = QWEN3_NEXT_MODEL + gsm8k_accuracy_thres = 0.93 + kl_div_thres = 0.0025 + other_args = [ + "--trust-remote-code", + "--speculative-algorithm", + "NEXTN", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--mem-fraction-static", + "0.8", + "--tp", + "4", + "--chunked-prefill-size", + "2048", + "--mamba-scheduler-strategy", + "no_buffer", + "--disable-radix-cache", + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_deepseek_v3_basic.py b/test/manual/8-gpu-models/test_deepseek_v3_basic.py similarity index 100% rename from test/registered/8-gpu-models/test_deepseek_v3_basic.py rename to test/manual/8-gpu-models/test_deepseek_v3_basic.py diff --git a/test/registered/8-gpu-models/test_dsa_models_basic.py b/test/manual/8-gpu-models/test_dsa_models_basic.py similarity index 100% rename from test/registered/8-gpu-models/test_dsa_models_basic.py rename to test/manual/8-gpu-models/test_dsa_models_basic.py diff --git a/test/registered/attention/test_fa3.py b/test/manual/attention/test_fa3.py similarity index 100% rename from test/registered/attention/test_fa3.py rename to test/manual/attention/test_fa3.py diff --git a/test/registered/attention/test_local_attn.py b/test/manual/attention/test_local_attn.py similarity index 100% rename from test/registered/attention/test_local_attn.py rename to test/manual/attention/test_local_attn.py diff --git a/test/registered/core/test_gpt_oss_1gpu.py b/test/manual/core/test_gpt_oss_1gpu.py similarity index 100% rename from test/registered/core/test_gpt_oss_1gpu.py rename to test/manual/core/test_gpt_oss_1gpu.py diff --git a/test/manual/distributed/test_dp_attention_archived.py b/test/manual/distributed/test_dp_attention_archived.py new file mode 100644 index 000000000..8b07ff280 --- /dev/null +++ b/test/manual/distributed/test_dp_attention_archived.py @@ -0,0 +1,101 @@ +"""Archived test classes split out of test/registered/distributed/test_dp_attention.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/distributed/test_dp_attention_archived.py`. +""" + +import unittest +from types import SimpleNamespace + +import requests + +from sglang.srt.utils import kill_process_tree +from sglang.test.kits.ebnf_constrained_kit import EBNFConstrainedMixin +from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin +from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + popen_launch_server, +) + + +class TestDPAttentionDP2TP2DeepseekV3MTP( + CustomTestCase, + JSONConstrainedMixin, + EBNFConstrainedMixin, + RegexConstrainedMixin, +): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--disable-radix", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "2", + "--speculative-eagle-topk", + "4", + "--speculative-num-draft-tokens", + "4", + "--speculative-draft-model-path", + DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, + "--tp-size", + "2", + "--enable-dp-attention", + "--dp-size", + "2", + ] + if not is_in_amd_ci(): + other_args += ["--mem-frac", "0.7"] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + requests.get(self.base_url + "/flush_cache") + + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["score"], 0.60) + + server_info = requests.get(self.base_url + "/server_info") + avg_spec_accept_length = server_info.json()["internal_states"][0][ + "avg_spec_accept_length" + ] + print( + f"###test_gsm8k (deepseek-v3 mtp + dp):\n" + f"accuracy={metrics['score']=:.3f}\n" + f"{avg_spec_accept_length=:.3f}\n" + ) + self.assertGreater(avg_spec_accept_length, 2.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/distributed/test_dp_attention_large.py b/test/manual/distributed/test_dp_attention_large.py similarity index 100% rename from test/registered/distributed/test_dp_attention_large.py rename to test/manual/distributed/test_dp_attention_large.py diff --git a/test/registered/eval/test_eval_accuracy_large.py b/test/manual/eval/test_eval_accuracy_large.py similarity index 100% rename from test/registered/eval/test_eval_accuracy_large.py rename to test/manual/eval/test_eval_accuracy_large.py diff --git a/test/registered/lora/test_lora_backend.py b/test/manual/lora/test_lora_backend.py similarity index 100% rename from test/registered/lora/test_lora_backend.py rename to test/manual/lora/test_lora_backend.py diff --git a/test/manual/mla/test_flashmla_archived.py b/test/manual/mla/test_flashmla_archived.py new file mode 100644 index 000000000..39266d3c5 --- /dev/null +++ b/test/manual/mla/test_flashmla_archived.py @@ -0,0 +1,73 @@ +"""Archived test classes split out of test/registered/mla/test_flashmla.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/mla/test_flashmla_archived.py`. +""" + +""" +Usage: +python3 test/registered/mla/test_flashmla.py +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_MODEL_NAME_FOR_TEST_MLA, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, +) + + +# FlashMLA attention backend tests with MTP speculative decoding +class TestFlashMLAAttnBackend(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "2", + "--attention-backend", + "flashmla", + ] + ) + # Use longer timeout for DeepGEMM JIT compilation which can take 10-20 minutes + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["score"], 0.60) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/mla/test_mla.py b/test/manual/mla/test_mla.py similarity index 100% rename from test/registered/mla/test_mla.py rename to test/manual/mla/test_mla.py diff --git a/test/registered/mla/test_mla_deepseek_v3.py b/test/manual/mla/test_mla_deepseek_v3.py similarity index 100% rename from test/registered/mla/test_mla_deepseek_v3.py rename to test/manual/mla/test_mla_deepseek_v3.py diff --git a/test/manual/mla/test_mla_flashinfer_archived.py b/test/manual/mla/test_mla_flashinfer_archived.py new file mode 100644 index 000000000..350e865db --- /dev/null +++ b/test/manual/mla/test_mla_flashinfer_archived.py @@ -0,0 +1,68 @@ +"""Archived test classes split out of test/registered/mla/test_mla_flashinfer.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/mla/test_mla_flashinfer_archived.py`. +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + + +# FlashInfer MLA backend tests with MTP speculative decoding +class TestFlashinferMLA(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--enable-torch-compile", + "--cuda-graph-max-bs", + "4", + "--attention-backend", + "flashinfer", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["score"], 0.615) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/manual/mla/test_mla_int8_deepseek_v3_archived.py b/test/manual/mla/test_mla_int8_deepseek_v3_archived.py new file mode 100644 index 000000000..8f083fb6c --- /dev/null +++ b/test/manual/mla/test_mla_int8_deepseek_v3_archived.py @@ -0,0 +1,113 @@ +"""Archived test classes split out of test/registered/mla/test_mla_int8_deepseek_v3.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/mla/test_mla_int8_deepseek_v3_archived.py`. +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + + +# DeepSeek-V3 INT8 quantization tests (channel and block INT8) +class TestMLADeepseekV3ChannelInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-channel-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "16", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreaterEqual(metrics["score"], 0.61) + + +@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") +class TestMLADeepseekV3BlockInt8(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = "lmsys/sglang-ci-dsv3-block-int8-test" + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = ["--trust-remote-code"] + if torch.cuda.is_available() and torch.version.cuda: + other_args.extend( + [ + "--cuda-graph-max-bs", + "16", + "--enable-torch-compile", + "--torch-compile-max-bs", + "2", + ] + ) + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=200, + num_threads=128, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["score"], 0.62) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/manual/models/test_nvidia_nemotron_3_nano_archived.py b/test/manual/models/test_nvidia_nemotron_3_nano_archived.py new file mode 100644 index 000000000..cc369abe7 --- /dev/null +++ b/test/manual/models/test_nvidia_nemotron_3_nano_archived.py @@ -0,0 +1,47 @@ +"""Archived test classes split out of test/registered/models/test_nvidia_nemotron_3_nano.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/models/test_nvidia_nemotron_3_nano_archived.py`. +""" + +import unittest + +from sglang.test.kits.lm_eval_kit import LMEvalMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase + +NEMOTRON_3_NANO_THINKING_ARGS = [ + "--trust-remote-code", + "--tool-call-parser", + "qwen3_coder", + "--reasoning-parser", + "deepseek-r1", +] + + +class TestNvidiaNemotron3Nano30BBF16(LMEvalMixin, DefaultServerBase): + """Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation.""" + + model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml" + other_args = [ + "--tp-size", + "2", + ] + NEMOTRON_3_NANO_THINKING_ARGS + + +class TestNvidiaNemotron3Nano30BBF16FlashInfer(LMEvalMixin, DefaultServerBase): + """Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation using flashinfer mamba backend.""" + + model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" + model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml" + other_args = [ + "--tp-size", + "2", + "--mamba-backend", + "flashinfer", + ] + NEMOTRON_3_NANO_THINKING_ARGS + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/models/test_nvidia_nemotron_nano_v2.py b/test/manual/models/test_nvidia_nemotron_nano_v2.py similarity index 100% rename from test/registered/models/test_nvidia_nemotron_nano_v2.py rename to test/manual/models/test_nvidia_nemotron_nano_v2.py diff --git a/test/registered/models/test_nvidia_nemotron_nano_v2_vl.py b/test/manual/models/test_nvidia_nemotron_nano_v2_vl.py similarity index 100% rename from test/registered/models/test_nvidia_nemotron_nano_v2_vl.py rename to test/manual/models/test_nvidia_nemotron_nano_v2_vl.py diff --git a/test/registered/models/test_qwen_models.py b/test/manual/models/test_qwen_models.py similarity index 100% rename from test/registered/models/test_qwen_models.py rename to test/manual/models/test_qwen_models.py diff --git a/test/registered/perf/test_bench_one_batch_1gpu.py b/test/manual/perf/test_bench_one_batch_1gpu.py similarity index 100% rename from test/registered/perf/test_bench_one_batch_1gpu.py rename to test/manual/perf/test_bench_one_batch_1gpu.py diff --git a/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py b/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py new file mode 100644 index 000000000..eb806da22 --- /dev/null +++ b/test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py @@ -0,0 +1,63 @@ +"""Archived test classes split out of test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu_archived.py`. +""" + +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + SimpleNamespace, + popen_launch_server, +) + + +# CI Registration +class TestPiecewiseCudaGraphInternVL25(CustomTestCase): + """Test piecewise CUDA graph with InternVL2.5-8B model""" + + @classmethod + def setUpClass(cls): + cls.model = "OpenGVLab/InternVL2_5-8B" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--enforce-piecewise-cuda-graph", + "--disable-radix-cache", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k_accuracy(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + num_examples=None, + num_threads=1024, + ) + + metrics = run_eval(args) + print(f"GSM8K Accuracy: {metrics['score']:.3f}") + + # Baseline (no piecewise CUDA graph): 0.571 — this eval uses 5-shot + # concatenated text via chat API, which scores lower than reported + # benchmarks (~77.8%) that use proper CoT chat format. The threshold + # is set 5% below observed to catch catastrophic regressions. + self.assertGreaterEqual(metrics["score"], 0.54) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/prefill_only/test_cross_encoder_models.py b/test/manual/prefill_only/test_cross_encoder_models.py similarity index 100% rename from test/registered/prefill_only/test_cross_encoder_models.py rename to test/manual/prefill_only/test_cross_encoder_models.py diff --git a/test/registered/prefill_only/test_encoder_embedding_models.py b/test/manual/prefill_only/test_encoder_embedding_models.py similarity index 100% rename from test/registered/prefill_only/test_encoder_embedding_models.py rename to test/manual/prefill_only/test_encoder_embedding_models.py diff --git a/test/registered/quant/test_autoround.py b/test/manual/quant/test_autoround.py similarity index 100% rename from test/registered/quant/test_autoround.py rename to test/manual/quant/test_autoround.py diff --git a/test/manual/quant/test_awq_archived.py b/test/manual/quant/test_awq_archived.py new file mode 100644 index 000000000..64a42acf4 --- /dev/null +++ b/test/manual/quant/test_awq_archived.py @@ -0,0 +1,57 @@ +"""Archived test classes split out of test/registered/quant/test_awq.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/quant/test_awq_archived.py`. +""" + +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_amd_ci, + popen_launch_server, +) + + +@unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs") +class TestAWQMarlinFloat16(CustomTestCase): + """ + Verify that the model can be loaded with float16 dtype and awq_marlin quantization + """ + + @classmethod + def setUpClass(cls): + cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ" + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--dtype", "float16", "--quantization", "awq_marlin"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_mmlu(self): + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="mmlu", + num_examples=64, + num_threads=32, + ) + + metrics = run_eval(args) + self.assertGreater(metrics["score"], 0.85) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_deepseek_v32_fp4_4gpu.py b/test/manual/quant/test_deepseek_v32_fp4_4gpu.py similarity index 100% rename from test/registered/quant/test_deepseek_v32_fp4_4gpu.py rename to test/manual/quant/test_deepseek_v32_fp4_4gpu.py diff --git a/test/registered/quant/test_eval_fp8_accuracy.py b/test/manual/quant/test_eval_fp8_accuracy.py similarity index 100% rename from test/registered/quant/test_eval_fp8_accuracy.py rename to test/manual/quant/test_eval_fp8_accuracy.py diff --git a/test/manual/quant/test_nvfp4_gemm_archived.py b/test/manual/quant/test_nvfp4_gemm_archived.py new file mode 100644 index 000000000..8468e754a --- /dev/null +++ b/test/manual/quant/test_nvfp4_gemm_archived.py @@ -0,0 +1,74 @@ +"""Archived test classes split out of test/registered/quant/test_nvfp4_gemm.py. + +Originally registered with `register_cuda_ci(...)`. Moved here as part of +the per-commit pruning effort to keep the code reachable manually. +Run with `python3 test/manual/quant/test_nvfp4_gemm_archived.py`. +""" + +import unittest +from types import SimpleNamespace +from urllib.parse import urlparse + +from sglang.srt.utils import get_device_sm, kill_process_tree +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + popen_launch_server, + try_cached_model, +) + +MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4" + + +class FP4GemmBase: + backend = None + + @classmethod + def setUpClass(cls): + if cls.backend is None: + raise NotImplementedError("Subclass must set 'backend' attribute") + cls.model = try_cached_model(MODEL_PATH) + cls.base_url = DEFAULT_URL_FOR_TEST + other_args = [ + "--trust-remote-code", + "--quantization", + "modelopt_fp4", + "--fp4-gemm-backend", + cls.backend, + ] + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=other_args, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + parsed_url = urlparse(self.base_url) + args = SimpleNamespace( + base_url=self.base_url, + model=self.model, + eval_name="gsm8k", + api="completion", + max_tokens=512, + num_examples=1319, + num_threads=200, + ) + metrics = run_eval(args) + print(metrics) + + self.assertGreater(metrics["score"], 0.64) + + +@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") +class TestFP4GemmAuto(FP4GemmBase, unittest.TestCase): + backend = "auto" + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/quant/test_quantization.py b/test/manual/quant/test_quantization.py similarity index 100% rename from test/registered/quant/test_quantization.py rename to test/manual/quant/test_quantization.py diff --git a/test/registered/scheduler/test_no_chunked_prefill.py b/test/manual/scheduler/test_no_chunked_prefill.py similarity index 100% rename from test/registered/scheduler/test_no_chunked_prefill.py rename to test/manual/scheduler/test_no_chunked_prefill.py diff --git a/test/registered/scheduler/test_no_overlap_scheduler.py b/test/manual/scheduler/test_no_overlap_scheduler.py similarity index 100% rename from test/registered/scheduler/test_no_overlap_scheduler.py rename to test/manual/scheduler/test_no_overlap_scheduler.py diff --git a/test/registered/spec/eagle/test_eagle3_basic.py b/test/manual/spec/eagle/test_eagle3_basic.py similarity index 100% rename from test/registered/spec/eagle/test_eagle3_basic.py rename to test/manual/spec/eagle/test_eagle3_basic.py diff --git a/test/registered/4-gpu-models/test_qwen35_models.py b/test/registered/4-gpu-models/test_qwen35_models.py index be125c321..7fc8a42ad 100644 --- a/test/registered/4-gpu-models/test_qwen35_models.py +++ b/test/registered/4-gpu-models/test_qwen35_models.py @@ -4,162 +4,25 @@ from types import SimpleNamespace import requests from sglang.srt.utils import kill_process_tree -from sglang.test.accuracy_test_runner import AccuracyTestParams from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin # This eval harness applies the chat_template, which is critical for qwen3.5 # to get good accuracy on gsm8k -from sglang.test.run_combined_tests import run_combined_tests from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - ModelLaunchSettings, popen_launch_server, ) -register_cuda_ci(est_time=768, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=260, suite="stage-c-test-4-gpu-b200") QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} -class TestQwen35FP4(CustomTestCase): - def test_gsm8k(self): - base_args = [ - "--tp-size", - "4", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "extra_buffer", - "--mamba-track-interval", - "128", - "--mamba-ssm-dtype", - "bfloat16", - "--max-running-requests", - "128", - "--reasoning-parser", - "qwen3", - "--attention-backend", - "trtllm_mha", - "--quantization", - "modelopt_fp4", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ] - - variants = [ - ModelLaunchSettings( - QWEN35_FP4_MODEL, - extra_args=base_args, - variant="Triton", - ), - # TODO: Fix this and re-enable it - # ModelLaunchSettings( - # QWEN35_FP4_MODEL, - # extra_args=base_args + ["--linear-attn-decode-backend", "flashinfer"], - # variant="FlashInfer", - # ), - ] - - run_combined_tests( - models=variants, - test_name="Qwen3.5-397B-A17B-NVFP4", - accuracy_params=AccuracyTestParams( - dataset="gsm8k", - baseline_accuracy=ACC_THRESHOLDS[QWEN35_FP4_MODEL]["gsm8k"], - num_examples=200, - num_threads=128, - max_tokens=16000, - thinking_mode="qwen3", - temperature=0.6, - top_p=0.95, - top_k=20, - ), - ) - - -class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): - reasoning_parser_name = "qwen3" - - @classmethod - def setUpClass(cls): - cls.model = QWEN35_FP4_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.init_reasoning_token_verifier() - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tp-size", - "4", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "extra_buffer", - "--mamba-track-interval", - "128", - "--mamba-ssm-dtype", - "bfloat16", - "--max-running-requests", - "128", - "--reasoning-parser", - "qwen3", - "--attention-backend", - "trtllm_mha", - "--quantization", - "modelopt_fp4", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.8", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - model=self.model, - eval_name="gsm8k", - num_shots=5, - num_examples=200, - max_tokens=16000, - num_threads=128, - repeat=1, - temperature=0.6, - top_p=0.95, - top_k=20, - base_url=self.base_url, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"]) - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 3.3) - - class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase): reasoning_parser_name = "qwen3" diff --git a/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py b/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py index 13ad4d0c3..23d1a8f26 100644 --- a/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py +++ b/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py @@ -6,37 +6,11 @@ from sglang.test.kits.kl_divergence_kit import KLDivergenceMixin from sglang.test.kits.prefix_cache_branching_kit import PrefixCacheBranchingMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase -register_cuda_ci(est_time=422, suite="stage-c-test-4-gpu-h100") +register_cuda_ci(est_time=290, suite="stage-c-test-4-gpu-h100") QWEN3_NEXT_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" -class TestQwen3NextMTP(GSM8KMixin, KLDivergenceMixin, DefaultServerBase): - model = QWEN3_NEXT_MODEL - gsm8k_accuracy_thres = 0.93 - kl_div_thres = 0.0025 - other_args = [ - "--trust-remote-code", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.8", - "--tp", - "4", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "no_buffer", - "--disable-radix-cache", - ] - - class TestQwen3NextMTPTopk( GSM8KMixin, KLDivergenceMixin, PrefixCacheBranchingMixin, DefaultServerBase ): diff --git a/test/registered/distributed/test_dp_attention.py b/test/registered/distributed/test_dp_attention.py index 53d86336a..01e8f6565 100644 --- a/test/registered/distributed/test_dp_attention.py +++ b/test/registered/distributed/test_dp_attention.py @@ -1,5 +1,4 @@ import unittest -from types import SimpleNamespace import requests @@ -12,20 +11,17 @@ from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.json_constrained_kit import JSONConstrainedMixin from sglang.test.kits.radix_cache_server_kit import run_radix_attention_test from sglang.test.kits.regex_constrained_kit import RegexConstrainedMixin -from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_IMAGE_URL, DEFAULT_MLA_MODEL_NAME_FOR_TEST, DEFAULT_MODEL_NAME_FOR_TEST_MLA, - DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, - is_in_amd_ci, popen_launch_server, ) -register_cuda_ci(est_time=524, suite="stage-b-test-2-gpu-large") +register_cuda_ci(est_time=420, suite="stage-b-test-2-gpu-large") class TestDPAttentionDP2TP2( @@ -140,77 +136,6 @@ class TestDPRetract( self.assertIsNone(self.process.poll()) -class TestDPAttentionDP2TP2DeepseekV3MTP( - CustomTestCase, - JSONConstrainedMixin, - EBNFConstrainedMixin, - RegexConstrainedMixin, -): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--trust-remote-code", - "--disable-radix", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "2", - "--speculative-eagle-topk", - "4", - "--speculative-num-draft-tokens", - "4", - "--speculative-draft-model-path", - DEFAULT_MODEL_NAME_FOR_TEST_MLA_NEXTN, - "--tp-size", - "2", - "--enable-dp-attention", - "--dp-size", - "2", - ] - if not is_in_amd_ci(): - other_args += ["--mem-frac", "0.7"] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.60) - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print( - f"###test_gsm8k (deepseek-v3 mtp + dp):\n" - f"accuracy={metrics['score']=:.3f}\n" - f"{avg_spec_accept_length=:.3f}\n" - ) - self.assertGreater(avg_spec_accept_length, 2.5) - - class TestDPAttentionDP2TP2VLM(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/registered/mla/test_flashmla.py b/test/registered/mla/test_flashmla.py index 97fd2e2ea..53c5964e4 100644 --- a/test/registered/mla/test_flashmla.py +++ b/test/registered/mla/test_flashmla.py @@ -14,7 +14,6 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( - DEFAULT_MODEL_NAME_FOR_TEST_MLA, DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -22,50 +21,7 @@ from sglang.test.test_utils import ( ) # FlashMLA attention backend tests with MTP speculative decoding -register_cuda_ci(est_time=314, suite="stage-b-test-1-gpu-large") - - -class TestFlashMLAAttnBackend(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.model = DEFAULT_MODEL_NAME_FOR_TEST_MLA - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--cuda-graph-max-bs", - "2", - "--attention-backend", - "flashmla", - ] - ) - # Use longer timeout for DeepGEMM JIT compilation which can take 10-20 minutes - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 2, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.60) +register_cuda_ci(est_time=160, suite="stage-b-test-1-gpu-large") class TestFlashMLAMTP(CustomTestCase): diff --git a/test/registered/mla/test_mla_flashinfer.py b/test/registered/mla/test_mla_flashinfer.py index 62d269b03..254d9367e 100644 --- a/test/registered/mla/test_mla_flashinfer.py +++ b/test/registered/mla/test_mla_flashinfer.py @@ -15,50 +15,7 @@ from sglang.test.test_utils import ( ) # FlashInfer MLA backend tests with MTP speculative decoding -register_cuda_ci(est_time=260, suite="stage-b-test-1-gpu-large") - - -class TestFlashinferMLA(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "lmsys/sglang-ci-dsv3-test" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--enable-torch-compile", - "--cuda-graph-max-bs", - "4", - "--attention-backend", - "flashinfer", - ] - ) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.615) +register_cuda_ci(est_time=130, suite="stage-b-test-1-gpu-large") class TestFlashinferMLAMTP(CustomTestCase): diff --git a/test/registered/mla/test_mla_int8_deepseek_v3.py b/test/registered/mla/test_mla_int8_deepseek_v3.py index 8a94544c8..65fddf8fb 100644 --- a/test/registered/mla/test_mla_int8_deepseek_v3.py +++ b/test/registered/mla/test_mla_int8_deepseek_v3.py @@ -16,53 +16,9 @@ from sglang.test.test_utils import ( ) # DeepSeek-V3 INT8 quantization tests (channel and block INT8) -register_cuda_ci(est_time=313, suite="stage-b-test-1-gpu-large") +register_cuda_ci(est_time=160, suite="stage-b-test-1-gpu-large") -class TestMLADeepseekV3ChannelInt8(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "lmsys/sglang-ci-dsv3-channel-int8-test" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--cuda-graph-max-bs", - "16", - "--enable-torch-compile", - "--torch-compile-max-bs", - "2", - ] - ) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreaterEqual(metrics["score"], 0.61) - - -@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") class TestDeepseekV3MTPChannelInt8(CustomTestCase): @classmethod def setUpClass(cls): @@ -126,49 +82,6 @@ class TestDeepseekV3MTPChannelInt8(CustomTestCase): @unittest.skipIf(is_in_ci(), "To reduce the CI execution time.") -class TestMLADeepseekV3BlockInt8(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = "lmsys/sglang-ci-dsv3-block-int8-test" - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = ["--trust-remote-code"] - if torch.cuda.is_available() and torch.version.cuda: - other_args.extend( - [ - "--cuda-graph-max-bs", - "16", - "--enable-torch-compile", - "--torch-compile-max-bs", - "2", - ] - ) - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(metrics) - - self.assertGreater(metrics["score"], 0.62) - - class TestDeepseekV3MTPBlockInt8(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/registered/models/test_nvidia_nemotron_3_nano.py b/test/registered/models/test_nvidia_nemotron_3_nano.py index b3bfc24bd..6fa052967 100644 --- a/test/registered/models/test_nvidia_nemotron_3_nano.py +++ b/test/registered/models/test_nvidia_nemotron_3_nano.py @@ -5,7 +5,7 @@ from sglang.test.kits.lm_eval_kit import LMEvalMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase register_cuda_ci( - est_time=564, + est_time=190, suite="stage-b-test-2-gpu-large", ) @@ -18,30 +18,6 @@ NEMOTRON_3_NANO_THINKING_ARGS = [ ] -class TestNvidiaNemotron3Nano30BBF16(LMEvalMixin, DefaultServerBase): - """Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation.""" - - model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml" - other_args = [ - "--tp-size", - "2", - ] + NEMOTRON_3_NANO_THINKING_ARGS - - -class TestNvidiaNemotron3Nano30BBF16FlashInfer(LMEvalMixin, DefaultServerBase): - """Test Nemotron-3-Nano-30B BF16 model with lm-eval GSM8K evaluation using flashinfer mamba backend.""" - - model = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" - model_config_name = "lm_eval_configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml" - other_args = [ - "--tp-size", - "2", - "--mamba-backend", - "flashinfer", - ] + NEMOTRON_3_NANO_THINKING_ARGS - - class TestNvidiaNemotron3Nano30BFP8(LMEvalMixin, DefaultServerBase): """Test Nemotron-3-Nano-30B FP8 model with lm-eval GSM8K evaluation.""" diff --git a/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py b/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py index 56e12fbd0..c6b92bb86 100644 --- a/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py +++ b/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py @@ -17,7 +17,7 @@ from sglang.test.test_utils import ( ) # CI Registration -register_cuda_ci(est_time=260, suite="stage-b-test-1-gpu-large") +register_cuda_ci(est_time=180, suite="stage-b-test-1-gpu-large") class TestPiecewiseCudaGraphQwen25VL(CustomTestCase): @@ -56,46 +56,6 @@ class TestPiecewiseCudaGraphQwen25VL(CustomTestCase): self.assertGreaterEqual(metrics["score"], 0.80) -class TestPiecewiseCudaGraphInternVL25(CustomTestCase): - """Test piecewise CUDA graph with InternVL2.5-8B model""" - - @classmethod - def setUpClass(cls): - cls.model = "OpenGVLab/InternVL2_5-8B" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--enforce-piecewise-cuda-graph", - "--disable-radix-cache", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k_accuracy(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - num_examples=None, - num_threads=1024, - ) - - metrics = run_eval(args) - print(f"GSM8K Accuracy: {metrics['score']:.3f}") - - # Baseline (no piecewise CUDA graph): 0.571 — this eval uses 5-shot - # concatenated text via chat API, which scores lower than reported - # benchmarks (~77.8%) that use proper CoT chat format. The threshold - # is set 5% below observed to catch catastrophic regressions. - self.assertGreaterEqual(metrics["score"], 0.54) - - class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): """Test piecewise CUDA graph with Qwen2.5-VL-3B-Instruct embedding model""" diff --git a/test/registered/quant/test_awq.py b/test/registered/quant/test_awq.py index ae13e8431..57450e665 100644 --- a/test/registered/quant/test_awq.py +++ b/test/registered/quant/test_awq.py @@ -13,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=226, suite="stage-b-test-1-gpu-large") +register_cuda_ci(est_time=160, suite="stage-b-test-1-gpu-large") register_amd_ci(est_time=200, suite="stage-b-test-1-gpu-large-amd") @@ -80,39 +80,5 @@ class TestAWQMarlinBfloat16(CustomTestCase): self.assertGreater(metrics["score"], 0.83) -@unittest.skipIf(is_in_amd_ci(), "AWQ Marlin is not supported on AMD GPUs") -class TestAWQMarlinFloat16(CustomTestCase): - """ - Verify that the model can be loaded with float16 dtype and awq_marlin quantization - """ - - @classmethod - def setUpClass(cls): - cls.model = "QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ" - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=["--dtype", "float16", "--quantization", "awq_marlin"], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_mmlu(self): - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="mmlu", - num_examples=64, - num_threads=32, - ) - - metrics = run_eval(args) - self.assertGreater(metrics["score"], 0.85) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/quant/test_nvfp4_gemm.py b/test/registered/quant/test_nvfp4_gemm.py index d91154c9c..2ff35bf7e 100644 --- a/test/registered/quant/test_nvfp4_gemm.py +++ b/test/registered/quant/test_nvfp4_gemm.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( try_cached_model, ) -register_cuda_ci(est_time=550, suite="stage-c-test-4-gpu-b200") +register_cuda_ci(est_time=420, suite="stage-c-test-4-gpu-b200") MODEL_PATH = "nvidia/Llama-3.1-8B-Instruct-NVFP4" @@ -61,11 +61,6 @@ class FP4GemmBase: self.assertGreater(metrics["score"], 0.64) -@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") -class TestFP4GemmAuto(FP4GemmBase, unittest.TestCase): - backend = "auto" - - @unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher") class TestFP4GemmFlashinferCutlass(FP4GemmBase, unittest.TestCase): backend = "flashinfer_cutlass"