diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl.py b/python/sglang/test/kits/unified_radix_cache_kit.py similarity index 58% rename from test/registered/radix_cache/test_unified_radix_cache_kl.py rename to python/sglang/test/kits/unified_radix_cache_kit.py index d7e4c5e05..e6fd24e58 100644 --- a/test/registered/radix_cache/test_unified_radix_cache_kl.py +++ b/python/sglang/test/kits/unified_radix_cache_kit.py @@ -1,25 +1,12 @@ import random -import unittest from types import SimpleNamespace from urllib.parse import urlparse -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kl_multiturn_utils import ( - get_input_ids, - make_mamba_decode_assert, - make_mamba_prefill_assert, test_input_output_logprobs_match_decode_cache_hit_helper, test_input_output_logprobs_match_helper, test_input_output_logprobs_match_prefill_cache_hit_helper, ) -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, -) def _random_suffixes(n, length, seed): @@ -28,18 +15,8 @@ def _random_suffixes(n, length, seed): return [[rng.randint(1, 30000) for _ in range(length)] for _ in range(n)] -MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" -MAMBA_CHUNK_SIZE = 64 -MAMBA_TRACK_INTERVAL = 128 - -SWA_MODEL = "openai/gpt-oss-20b" -FULL_MODEL = "Qwen/Qwen3-32B" - -register_cuda_ci(est_time=760, stage="base-c", runner_config="4-gpu-h100") - - class UnifiedRadixTreeTestMixin: - """Mixin: gsm8k、mmlu and multi-turn KL tests with multi-branch interleaving.""" + """Mixin: gsm8k, mmlu and multi-turn KL tests with multi-branch interleaving.""" kl_threshold: float = 0.003 max_new_tokens: int = 512 @@ -168,112 +145,3 @@ class UnifiedRadixTreeTestMixin: request_batch_size=self.decode_hit_request_batch_size, inter_batch_delay_s=self.decode_hit_inter_batch_delay_s, ) - - -class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): - """Full attention.""" - - kl_threshold = 0.0025 - - @classmethod - def setUpClass(cls): - cls.model = FULL_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tp-size", - "4", - "--mem-fraction-static", - "0.80", - "--page-size", - "64", - ], - env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, - ) - cls.input_ids = get_input_ids(cls.model, num_samples=18) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): - """Mamba hybrid + UnifiedRadixCache.""" - - kl_threshold = 0.003 - prefill_cache_assert = staticmethod( - make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE) - ) - decode_cache_assert = staticmethod( - make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL) - ) - - @classmethod - def setUpClass(cls): - cls.model = MAMBA_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tp-size", - "4", - "--chunked-prefill-size", - "2048", - "--mem-fraction-static", - "0.85", - "--mamba-scheduler-strategy", - "extra_buffer", - "--mamba-track-interval", - str(MAMBA_TRACK_INTERVAL), - ], - env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, - ) - cls.input_ids = get_input_ids(cls.model, num_samples=18) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): - """SWA hybrid + UnifiedRadixCache.""" - - kl_threshold = 0.03 - gsm8k_threshold = 0.7 - mmlu_threshold = 0.7 - - @unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough") - def test_mmlu(self): - super().test_mmlu() - - @classmethod - def setUpClass(cls): - cls.model = SWA_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tp-size", - "4", - "--mem-fraction-static", - "0.7", - "--disable-piecewise-cuda-graph", - ], - env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, - ) - cls.input_ids = get_input_ids(cls.model, num_samples=18) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/manual/core/test_dsv4_hicache_swa_translation_cache.py b/test/manual/core/test_dsv4_hicache_swa_translation_cache.py index 5a83c2e5b..b888149e8 100644 --- a/test/manual/core/test_dsv4_hicache_swa_translation_cache.py +++ b/test/manual/core/test_dsv4_hicache_swa_translation_cache.py @@ -23,7 +23,6 @@ Test strategy: import unittest -from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin from test_unified_radix_cache_kl_hicache import ( DSV4_FLASH_LAUNCH_TIMEOUT, DSV4_FLASH_MODEL, @@ -31,6 +30,7 @@ from test_unified_radix_cache_kl_hicache import ( ) from sglang.srt.utils import kill_process_tree +from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin from sglang.test.kl_multiturn_utils import get_input_ids from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, diff --git a/test/registered/4-gpu-models/test_qwen35_models.py b/test/registered/4-gpu-models/test_qwen35_models.py deleted file mode 100644 index 434f690c5..000000000 --- a/test/registered/4-gpu-models/test_qwen35_models.py +++ /dev/null @@ -1,105 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.reasoning_kit import ReasoningTokenUsageMixin - -# This eval harness applies the chat_template, which is critical for qwen3.5 -# to get good accuracy on gsm8k -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) - -register_cuda_ci(est_time=370, stage="base-c", runner_config="4-gpu-b200") - -QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" -ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} - - -class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase): - reasoning_parser_name = "qwen3" - - @classmethod - def setUpClass(cls): - cls.model = QWEN35_FP4_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.init_reasoning_token_verifier() - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=[ - "--tp-size", - "4", - "--chunked-prefill-size", - "2048", - "--mamba-scheduler-strategy", - "extra_buffer", - "--mamba-track-interval", - "128", - "--mamba-ssm-dtype", - "bfloat16", - "--max-running-requests", - "128", - "--reasoning-parser", - "qwen3", - "--attention-backend", - "trtllm_mha", - "--quantization", - "modelopt_fp4", - "--speculative-algorithm", - "NEXTN", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--mem-fraction-static", - "0.8", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - args = SimpleNamespace( - model=self.model, - eval_name="gsm8k", - num_shots=5, - num_examples=200, - max_tokens=16000, - num_threads=128, - repeat=1, - temperature=0.6, - top_p=0.95, - top_k=20, - base_url=self.base_url, - host="http://127.0.0.1", - port=int(self.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - self.assertGreaterEqual(metrics["score"], ACC_THRESHOLDS[self.model]["gsm8k"]) - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - self.assertGreater(avg_spec_accept_length, 3.3) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/8-gpu-models/test_deepseek_v3_mtp.py b/test/registered/8-gpu-models/test_deepseek_v3_mtp.py deleted file mode 100644 index 9bcb6ebe1..000000000 --- a/test/registered/8-gpu-models/test_deepseek_v3_mtp.py +++ /dev/null @@ -1,110 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.send_one import BenchArgs, send_one_prompt -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_amd_ci, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200") - -FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" - - -class TestDeepseekV3MTP(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = FULL_DEEPSEEK_V3_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "8", - "--trust-remote-code", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--model-loader-extra-config", - '{"enable_multithread_load": true, "num_threads": 64}', - ] - if not is_in_amd_ci(): - other_args += ["--mem-frac", "0.7"] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v3 mtp)\n" - f'{metrics["score"]=:.3f}\n' - f"{avg_spec_accept_length=:.2f}\n" - ) - self.assertGreater(metrics["score"], 0.935) - self.assertGreater(avg_spec_accept_length, 2.8) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - acc_length, speed = send_one_prompt(args) - - print(f"{acc_length=:.2f} {speed=:.2f}") - - if is_in_ci(): - write_github_step_summary( - f"### test_bs_1_speed (deepseek-v3 mtp)\n" - f"{acc_length=:.2f}\n" - f"{speed=:.2f} token/s\n" - ) - self.assertGreater(acc_length, 2.8) - if is_in_amd_ci(): - self.assertGreater(speed, 15) - else: - self.assertGreater(speed, 130) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/4-gpu-models/test_qwen35_hicache.py b/test/registered/hicache/test_qwen35_hicache.py similarity index 100% rename from test/registered/4-gpu-models/test_qwen35_hicache.py rename to test/registered/hicache/test_qwen35_hicache.py diff --git a/test/registered/models_e2e/test_deepseek_v32_fp4_mtp_dp.py b/test/registered/models_e2e/test_deepseek_v32_fp4_mtp_dp.py new file mode 100644 index 000000000..cb4a1c6f9 --- /dev/null +++ b/test/registered/models_e2e/test_deepseek_v32_fp4_mtp_dp.py @@ -0,0 +1,68 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.test_utils import is_in_ci, write_github_step_summary + +register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200") + +DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4" + + +class TestDeepseekV32FP4DPSpec(GSM8KMixin, DefaultServerBase): + model = DSV32_FP4_MODEL + timeout = 1200 + other_args = [ + "--tp", + "4", + "--dp", + "4", + "--enable-dp-attention", + "--attention-backend", + "dsa", + "--moe-runner-backend", + "flashinfer_trtllm", + "--quantization", + "modelopt_fp4", + "--tool-call-parser", + "deepseekv32", + "--reasoning-parser", + "deepseek-v3", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ] + + gsm8k_accuracy_thres = 0.93 + gsm8k_num_questions = 500 + gsm8k_num_threads = 500 + gsm8k_num_shots = 20 + gsm8k_accept_length_thres = 2.7 + + def test_z_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 mtp dp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, 2.7) + self.assertGreater(speed, 90) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/models_e2e/test_deepseek_v32_fp4_mtp_tp.py b/test/registered/models_e2e/test_deepseek_v32_fp4_mtp_tp.py new file mode 100644 index 000000000..c0eaaf230 --- /dev/null +++ b/test/registered/models_e2e/test_deepseek_v32_fp4_mtp_tp.py @@ -0,0 +1,65 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.test_utils import is_in_ci, write_github_step_summary + +register_cuda_ci(est_time=400, stage="base-c", runner_config="4-gpu-b200") + +DSV32_FP4_MODEL = "nvidia/DeepSeek-V3.2-NVFP4" + + +class TestDeepseekV32FP4TPSpec(GSM8KMixin, DefaultServerBase): + model = DSV32_FP4_MODEL + timeout = 1200 + other_args = [ + "--tp", + "4", + "--attention-backend", + "dsa", + "--moe-runner-backend", + "flashinfer_trtllm", + "--quantization", + "modelopt_fp4", + "--tool-call-parser", + "deepseekv32", + "--reasoning-parser", + "deepseek-v3", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + ] + + gsm8k_accuracy_thres = 0.93 + gsm8k_num_questions = 500 + gsm8k_num_threads = 500 + gsm8k_num_shots = 20 + gsm8k_accept_length_thres = 2.7 + + def test_z_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v32 mtp tp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, 2.7) + self.assertGreater(speed, 150) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/models_e2e/test_deepseek_v3_fp4.py b/test/registered/models_e2e/test_deepseek_v3_fp4.py new file mode 100644 index 000000000..a60db1025 --- /dev/null +++ b/test/registered/models_e2e/test_deepseek_v3_fp4.py @@ -0,0 +1,41 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase + +# Per-commit: SymmetricMemory variant only. +# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py +# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py +register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200") + +FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" + + +class TestDeepseekV3FP4SymmetricMemory(GSM8KMixin, DefaultServerBase): + model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH + timeout = 1200 + other_args = [ + "--tp", + "4", + "--attention-backend", + "trtllm_mla", + "--moe-runner-backend", + "flashinfer_trtllm", + "--quantization", + "modelopt_fp4", + "--kv-cache-dtype", + "fp8_e4m3", + "--model-loader-extra-config", + '{"enable_multithread_load": true,"num_threads": 64}', + "--enable-symm-mem", + ] + + gsm8k_accuracy_thres = 0.93 + gsm8k_num_questions = 1319 + gsm8k_num_threads = 1319 + gsm8k_num_shots = 8 + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/models_e2e/test_deepseek_v3_mtp.py b/test/registered/models_e2e/test_deepseek_v3_mtp.py new file mode 100644 index 000000000..fb77d5bae --- /dev/null +++ b/test/registered/models_e2e/test_deepseek_v3_mtp.py @@ -0,0 +1,67 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.send_one import BenchArgs, send_one_prompt +from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + is_in_amd_ci, + is_in_ci, + write_github_step_summary, +) + +register_cuda_ci(est_time=300, stage="base-c", runner_config="8-gpu-h200") + +FULL_DEEPSEEK_V3_MODEL_PATH = "deepseek-ai/DeepSeek-V3-0324" + +_OTHER_ARGS = [ + "--tp", + "8", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', +] +if not is_in_amd_ci(): + _OTHER_ARGS += ["--mem-frac", "0.7"] + + +class TestDeepseekV3MTP(GSM8KMixin, DefaultServerBase): + model = FULL_DEEPSEEK_V3_MODEL_PATH + other_args = _OTHER_ARGS + timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 5 + + gsm8k_accuracy_thres = 0.935 + gsm8k_accept_length_thres = 2.8 + + # `test_z_bs_1_speed` runs after `test_gsm8k` (alphabetical) so it + # measures steady-state speed on a warmed server. + def test_z_bs_1_speed(self): + args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) + acc_length, speed = send_one_prompt(args) + + print(f"{acc_length=:.2f} {speed=:.2f}") + + if is_in_ci(): + write_github_step_summary( + f"### test_bs_1_speed (deepseek-v3 mtp)\n" + f"{acc_length=:.2f}\n" + f"{speed=:.2f} token/s\n" + ) + self.assertGreater(acc_length, 2.8) + if is_in_amd_ci(): + self.assertGreater(speed, 15) + else: + self.assertGreater(speed, 130) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py b/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py new file mode 100644 index 000000000..db0d4ddc0 --- /dev/null +++ b/test/registered/models_e2e/test_gpt_oss_4gpu_bf16.py @@ -0,0 +1,23 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.gpt_oss_common import BaseTestGptOss + +register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200") + + +class TestGptOss4GpuBf16(BaseTestGptOss): + def test_bf16_120b(self): + self.run_test( + model_variant="120b", + quantization="bf16", + expected_score_of_reasoning_effort={ + "low": 0.58, + }, + other_args=["--tp", "4", "--cuda-graph-max-bs", "200"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py b/test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py similarity index 55% rename from test/registered/4-gpu-models/test_gpt_oss_4gpu.py rename to test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py index c792c4d56..8475c7eab 100644 --- a/test/registered/4-gpu-models/test_gpt_oss_4gpu.py +++ b/test/registered/models_e2e/test_gpt_oss_4gpu_mxfp4.py @@ -3,21 +3,11 @@ import unittest from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.gpt_oss_common import BaseTestGptOss -register_cuda_ci(est_time=392, stage="base-c", runner_config="4-gpu-h100") -register_cuda_ci(est_time=350, stage="base-c", runner_config="4-gpu-b200") +register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-h100") +register_cuda_ci(est_time=220, stage="base-c", runner_config="4-gpu-b200") -class TestGptOss4Gpu(BaseTestGptOss): - def test_bf16_120b(self): - self.run_test( - model_variant="120b", - quantization="bf16", - expected_score_of_reasoning_effort={ - "low": 0.58, - }, - other_args=["--tp", "4", "--cuda-graph-max-bs", "200"], - ) - +class TestGptOss4GpuMxfp4(BaseTestGptOss): def test_mxfp4_120b(self): self.run_test( model_variant="120b", diff --git a/test/registered/models_e2e/test_mimo_v2.py b/test/registered/models_e2e/test_mimo_v2.py new file mode 100644 index 000000000..8f1898a8b --- /dev/null +++ b/test/registered/models_e2e/test_mimo_v2.py @@ -0,0 +1,47 @@ +import unittest + +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase + +register_cuda_ci(est_time=400, stage="base-c", runner_config="8-gpu-h200") + +MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5" +MIMO_V2_OTHER_ARGS = [ + "--tp", + "8", + "--dp", + "2", + "--enable-dp-attention", + "--mm-enable-dp-encoder", + "--attention-backend", + "fa3", + "--mm-attention-backend", + "fa3", + "--reasoning-parser", + "mimo", +] +MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [ + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--enable-multi-layer-eagle", +] + + +class TestMiMoV2(GSM8KMixin, MMMUServerBase): + gsm8k_accuracy_thres = 0.75 + gsm8k_accept_length_thres = 2.5 + model = MIMO_V2_MODEL + mem_fraction_static = 0.65 + server_api_key = None + other_args = MIMO_V2_MTP_OTHER_ARGS + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_mimo_models.py b/test/registered/models_e2e/test_mimo_v2_flash.py similarity index 59% rename from test/registered/8-gpu-models/test_mimo_models.py rename to test/registered/models_e2e/test_mimo_v2_flash.py index 6d4c65230..6615b204e 100644 --- a/test/registered/8-gpu-models/test_mimo_models.py +++ b/test/registered/models_e2e/test_mimo_v2_flash.py @@ -4,9 +4,8 @@ from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin from sglang.test.kits.spec_decoding_kit import SpecDecodingMixin from sglang.test.server_fixtures.default_fixture import DefaultServerBase -from sglang.test.server_fixtures.mmmu_fixture import MMMUServerBase -register_cuda_ci(est_time=500, stage="base-c", runner_config="8-gpu-h200") +register_cuda_ci(est_time=350, stage="base-c", runner_config="8-gpu-h200") class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): @@ -49,42 +48,5 @@ class TestMiMoV2Flash(GSM8KMixin, SpecDecodingMixin, DefaultServerBase): accept_length_thres = 3.2 -MIMO_V2_MODEL = "XiaomiMiMo/MiMo-V2.5" -MIMO_V2_OTHER_ARGS = [ - "--tp", - "8", - "--dp", - "2", - "--enable-dp-attention", - "--mm-enable-dp-encoder", - "--attention-backend", - "fa3", - "--mm-attention-backend", - "fa3", - "--reasoning-parser", - "mimo", -] -MIMO_V2_MTP_OTHER_ARGS = MIMO_V2_OTHER_ARGS + [ - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--enable-multi-layer-eagle", -] - - -class TestMiMoV2(GSM8KMixin, MMMUServerBase): - gsm8k_accuracy_thres = 0.75 - gsm8k_accept_length_thres = 2.5 - model = MIMO_V2_MODEL - mem_fraction_static = 0.65 - server_api_key = None - other_args = MIMO_V2_MTP_OTHER_ARGS - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/8-gpu-models/test_minimax_m25_basic.py b/test/registered/models_e2e/test_minimax_m25_basic.py similarity index 100% rename from test/registered/8-gpu-models/test_minimax_m25_basic.py rename to test/registered/models_e2e/test_minimax_m25_basic.py diff --git a/test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16.py b/test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16.py new file mode 100644 index 000000000..2b537eee8 --- /dev/null +++ b/test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16.py @@ -0,0 +1,69 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.run_eval import run_eval +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200") + +NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" + +NEMOTRON_3_SUPER_BF16_ARGS = [ + "--tp-size", + "8", + "--trust-remote-code", + "--reasoning-parser", + "nemotron_3", + "--tool-call-parser", + "qwen3_coder", + "--disable-radix-cache", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 50}', +] + + +class TestNvidiaNemotron3SuperBF16(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.model = NEMOTRON_3_SUPER_BF16_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=NEMOTRON_3_SUPER_BF16_ARGS, + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + args = SimpleNamespace( + model=self.model, + eval_name="gsm8k", + num_shots=5, + num_examples=200, + max_tokens=16000, + num_threads=200, + repeat=1, + temperature=1.0, + top_p=0.95, + base_url=self.base_url, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["score"], 0.96) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py b/test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16_mtp.py similarity index 58% rename from test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py rename to test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16_mtp.py index dc395e266..1d9f2692f 100644 --- a/test/registered/8-gpu-models/test_nvidia_nemotron_3_super_bf16.py +++ b/test/registered/models_e2e/test_nvidia_nemotron_3_super_bf16_mtp.py @@ -11,7 +11,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=370, stage="extra-b", runner_config="8-gpu-h200") +register_cuda_ci(est_time=200, stage="extra-b", runner_config="8-gpu-h200") NEMOTRON_3_SUPER_BF16_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" @@ -44,46 +44,6 @@ MTP_ARGS = [ ] -def _run_gsm8k(test_case): - args = SimpleNamespace( - model=test_case.model, - eval_name="gsm8k", - num_shots=5, - num_examples=200, - max_tokens=16000, - num_threads=200, - repeat=1, - temperature=1.0, - top_p=0.95, - base_url=test_case.base_url, - host="http://127.0.0.1", - port=int(test_case.base_url.split(":")[-1]), - ) - metrics = run_eval(args) - print(f"{metrics=}") - test_case.assertGreaterEqual(metrics["score"], 0.96) - - -class TestNvidiaNemotron3SuperBF16(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = NEMOTRON_3_SUPER_BF16_MODEL - cls.base_url = DEFAULT_URL_FOR_TEST - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - other_args=NEMOTRON_3_SUPER_BF16_ARGS, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - _run_gsm8k(self) - - class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase): @classmethod def setUpClass(cls): @@ -101,7 +61,23 @@ class TestNvidiaNemotron3SuperBF16MTP(CustomTestCase): kill_process_tree(cls.process.pid) def test_gsm8k(self): - _run_gsm8k(self) + args = SimpleNamespace( + model=self.model, + eval_name="gsm8k", + num_shots=5, + num_examples=200, + max_tokens=16000, + num_threads=200, + repeat=1, + temperature=1.0, + top_p=0.95, + base_url=self.base_url, + host="http://127.0.0.1", + port=int(self.base_url.split(":")[-1]), + ) + metrics = run_eval(args) + print(f"{metrics=}") + self.assertGreaterEqual(metrics["score"], 0.96) if __name__ == "__main__": diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_flashinfer.py b/test/registered/models_e2e/test_qwen35_fp4_flashinfer.py similarity index 100% rename from test/registered/4-gpu-models/test_qwen35_fp4_flashinfer.py rename to test/registered/models_e2e/test_qwen35_fp4_flashinfer.py diff --git a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py b/test/registered/models_e2e/test_qwen35_fp4_mtp.py similarity index 97% rename from test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py rename to test/registered/models_e2e/test_qwen35_fp4_mtp.py index 6e9c8ca6a..d6b57a646 100644 --- a/test/registered/4-gpu-models/test_qwen35_fp4_mtp_v2.py +++ b/test/registered/models_e2e/test_qwen35_fp4_mtp.py @@ -21,7 +21,7 @@ QWEN35_FP4_MODEL = "nvidia/Qwen3.5-397B-A17B-NVFP4" ACC_THRESHOLDS = {QWEN35_FP4_MODEL: {"gsm8k": 0.95}} -class TestQwen35FP4MTPV2(ReasoningTokenUsageMixin, CustomTestCase): +class TestQwen35FP4MTP(ReasoningTokenUsageMixin, CustomTestCase): reasoning_parser_name = "qwen3" @classmethod diff --git a/test/registered/4-gpu-models/test_qwen3_next_models_mtp.py b/test/registered/models_e2e/test_qwen3_next_models_mtp.py similarity index 100% rename from test/registered/4-gpu-models/test_qwen3_next_models_mtp.py rename to test/registered/models_e2e/test_qwen3_next_models_mtp.py diff --git a/test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py b/test/registered/models_e2e/test_step3p5_flash_chain_mtp.py similarity index 74% rename from test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py rename to test/registered/models_e2e/test_step3p5_flash_chain_mtp.py index 2b375888c..223d07fa7 100644 --- a/test/registered/8-gpu-models/test_step3p5_flash_chain_mtp.py +++ b/test/registered/models_e2e/test_step3p5_flash_chain_mtp.py @@ -1,28 +1,20 @@ import unittest -from types import SimpleNamespace import numpy as np import requests from sglang.srt.environ import envs -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.server_fixtures.default_fixture import DefaultServerBase +from sglang.test.test_utils import DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH register_cuda_ci(est_time=480, stage="extra-b", runner_config="8-gpu-h200") STEP3P5_FLASH_MODEL_PATH = "stepfun-ai/Step-3.5-Flash" -class TestStep3p5FlashChainMTP(CustomTestCase): +class TestStep3p5FlashChainMTP(GSM8KMixin, DefaultServerBase): """Chain-style multi-layer EAGLE speculative decoding on Step-3.5-Flash. Step3p5ForCausalLM auto-enables multi-layer EAGLE and spec v2 when @@ -31,75 +23,38 @@ class TestStep3p5FlashChainMTP(CustomTestCase): automatically for the Step3p5MTP draft architecture. """ + model = STEP3P5_FLASH_MODEL_PATH + timeout = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3 + other_args = [ + "--tp", + "8", + "--trust-remote-code", + "--speculative-algorithm", + "EAGLE", + "--speculative-num-steps", + "3", + "--speculative-eagle-topk", + "1", + "--speculative-num-draft-tokens", + "4", + "--attention-backend", + "fa3", + "--enable-multi-layer-eagle", + "--mem-fraction-static", + "0.75", + "--chunked-prefill-size", + "4096", + "--model-loader-extra-config", + '{"enable_multithread_load": true, "num_threads": 64}', + ] + + gsm8k_accuracy_thres = 0.83 + gsm8k_accept_length_thres = 2.6 + @classmethod def setUpClass(cls): - cls.model = STEP3P5_FLASH_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "8", - "--trust-remote-code", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--attention-backend", - "fa3", - "--enable-multi-layer-eagle", - "--mem-fraction-static", - "0.75", - "--chunked-prefill-size", - "4096", - "--model-loader-extra-config", - '{"enable_multithread_load": true, "num_threads": 64}', - ] with envs.SGLANG_ENABLE_SPEC_V2.override(True): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - if hasattr(cls, "process") and cls.process: - kill_process_tree(cls.process.pid) - - def test_gsm8k(self): - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=200, - num_threads=128, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - print(f"{metrics=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (step-3.5-flash chain mtp)\n" - f'{metrics["score"]=:.3f}\n' - f"{avg_spec_accept_length=:.2f}\n" - ) - self.assertGreater(metrics["score"], 0.83) - self.assertGreater(avg_spec_accept_length, 2.6) + super().setUpClass() def test_logprob_spec_v2_match(self): """Verify spec v2 decode logprobs match prefill scoring logprobs. diff --git a/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py b/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py deleted file mode 100644 index 77d11b34e..000000000 --- a/test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py +++ /dev/null @@ -1,212 +0,0 @@ -import unittest -from types import SimpleNamespace - -import requests - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.send_one import BenchArgs, send_one_prompt -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -register_cuda_ci( - est_time=690, - stage="base-c", - runner_config="4-gpu-b200", -) - -FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3.2-NVFP4" -SERVER_LAUNCH_TIMEOUT = 1200 - - -class TestDeepseekV32FP4DPSpecV2(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "4", - "--dp", - "4", - "--enable-dp-attention", - "--attention-backend", - "dsa", - "--moe-runner-backend", - "flashinfer_trtllm", - "--quantization", - "modelopt_fp4", - "--tool-call-parser", - "deepseekv32", - "--reasoning-parser", - "deepseek-v3", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=500, - num_threads=500, - num_shots=20, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["score"]=:.3f}\n' - f"{avg_spec_accept_length=:.2f}\n" - ) - self.assertGreater(metrics["score"], 0.93) - self.assertGreater(avg_spec_accept_length, 2.7) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - acc_length, speed = send_one_prompt(args) - - print(f"{acc_length=:.2f} {speed=:.2f}") - - if is_in_ci(): - write_github_step_summary( - f"### test_bs_1_speed (deepseek-v32 mtp)\n" - f"{acc_length=:.2f}\n" - f"{speed=:.2f} token/s\n" - ) - - self.assertGreater(acc_length, 2.7) - self.assertGreater(speed, 90) - - -class TestDeepseekV32FP4TPSpecV2(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "4", - "--attention-backend", - "dsa", - "--moe-runner-backend", - "flashinfer_trtllm", - "--quantization", - "modelopt_fp4", - "--tool-call-parser", - "deepseekv32", - "--reasoning-parser", - "deepseek-v3", - "--speculative-algorithm", - "EAGLE", - "--speculative-num-steps", - "3", - "--speculative-eagle-topk", - "1", - "--speculative-num-draft-tokens", - "4", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - requests.get(self.base_url + "/flush_cache") - - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=500, - num_threads=500, - num_shots=20, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - server_info = requests.get(self.base_url + "/server_info") - avg_spec_accept_length = server_info.json()["internal_states"][0][ - "avg_spec_accept_length" - ] - print(f"{avg_spec_accept_length=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v32 mtp)\n" - f'{metrics["score"]=:.3f}\n' - f"{avg_spec_accept_length=:.2f}\n" - ) - self.assertGreater(metrics["score"], 0.93) - self.assertGreater(avg_spec_accept_length, 2.7) - - def test_bs_1_speed(self): - args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048) - acc_length, speed = send_one_prompt(args) - - print(f"{acc_length=:.2f} {speed=:.2f}") - - if is_in_ci(): - write_github_step_summary( - f"### test_bs_1_speed (deepseek-v32 mtp)\n" - f"{acc_length=:.2f}\n" - f"{speed=:.2f} token/s\n" - ) - - self.assertGreater(acc_length, 2.7) - self.assertGreater(speed, 150) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py b/test/registered/quant/test_deepseek_v3_fp4_4gpu.py deleted file mode 100644 index 927636018..000000000 --- a/test/registered/quant/test_deepseek_v3_fp4_4gpu.py +++ /dev/null @@ -1,80 +0,0 @@ -import unittest -from types import SimpleNamespace - -from sglang.srt.utils import kill_process_tree -from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.run_eval import run_eval -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - is_in_ci, - popen_launch_server, - write_github_step_summary, -) - -# Per-commit: SymmetricMemory variant only. -# - TestDeepseekV3FP4 (TRTLLM) archived to test/manual/quant/test_deepseek_v3_fp4_4gpu_trtllm.py -# - TestDeepseekV3FP4CutlassMoE moved to test_deepseek_v3_fp4_4gpu_extra.py -register_cuda_ci(est_time=960, stage="base-c", runner_config="4-gpu-b200") - -FULL_DEEPSEEK_V3_FP4_MODEL_PATH = "nvidia/DeepSeek-V3-0324-FP4" -SERVER_LAUNCH_TIMEOUT = 1200 - - -class TestDeepseekV3FP4SymmetricMemory(CustomTestCase): - @classmethod - def setUpClass(cls): - cls.model = FULL_DEEPSEEK_V3_FP4_MODEL_PATH - cls.base_url = DEFAULT_URL_FOR_TEST - other_args = [ - "--tp", - "4", - "--attention-backend", - "trtllm_mla", - "--moe-runner-backend", - "flashinfer_trtllm", - "--quantization", - "modelopt_fp4", - "--kv-cache-dtype", - "fp8_e4m3", - "--model-loader-extra-config", - '{"enable_multithread_load": true,"num_threads": 64}', - "--enable-symm-mem", - ] - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=SERVER_LAUNCH_TIMEOUT, - other_args=other_args, - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) - - def test_a_gsm8k( - self, - ): # Append an "a" to make this test run first (alphabetically) to warm up the server - args = SimpleNamespace( - base_url=self.base_url, - model=self.model, - eval_name="gsm8k", - api="completion", - max_tokens=512, - num_examples=1319, - num_threads=1319, - num_shots=8, - ) - metrics = run_eval(args) - print(f"{metrics=}") - - if is_in_ci(): - write_github_step_summary( - f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n' - ) - - self.assertGreater(metrics["score"], 0.93) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_full.py b/test/registered/radix_cache/test_unified_radix_cache_kl_full.py new file mode 100644 index 000000000..cb8e6e48d --- /dev/null +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_full.py @@ -0,0 +1,50 @@ +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin +from sglang.test.kl_multiturn_utils import get_input_ids +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100") + +FULL_MODEL = "Qwen/Qwen3-32B" + + +class TestUnifiedFullRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): + """Full attention.""" + + kl_threshold = 0.0025 + + @classmethod + def setUpClass(cls): + cls.model = FULL_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--mem-fraction-static", + "0.80", + "--page-size", + "64", + ], + env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, + ) + cls.input_ids = get_input_ids(cls.model, num_samples=18) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py index 1e0344b72..9a108c9c7 100644 --- a/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_hicache.py @@ -1,9 +1,8 @@ import unittest -from test_unified_radix_cache_kl import UnifiedRadixTreeTestMixin - from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin from sglang.test.kl_multiturn_utils import ( get_input_ids, make_mamba_decode_assert, diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_mamba.py b/test/registered/radix_cache/test_unified_radix_cache_kl_mamba.py new file mode 100644 index 000000000..2e540881f --- /dev/null +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_mamba.py @@ -0,0 +1,66 @@ +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin +from sglang.test.kl_multiturn_utils import ( + get_input_ids, + make_mamba_decode_assert, + make_mamba_prefill_assert, +) +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100") + +MAMBA_MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct" +MAMBA_CHUNK_SIZE = 64 +MAMBA_TRACK_INTERVAL = 128 + + +class TestUnifiedMambaRadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): + """Mamba hybrid + UnifiedRadixCache.""" + + kl_threshold = 0.003 + prefill_cache_assert = staticmethod( + make_mamba_prefill_assert(chunk_size=MAMBA_CHUNK_SIZE) + ) + decode_cache_assert = staticmethod( + make_mamba_decode_assert(track_interval=MAMBA_TRACK_INTERVAL) + ) + + @classmethod + def setUpClass(cls): + cls.model = MAMBA_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--chunked-prefill-size", + "2048", + "--mem-fraction-static", + "0.85", + "--mamba-scheduler-strategy", + "extra_buffer", + "--mamba-track-interval", + str(MAMBA_TRACK_INTERVAL), + ], + env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, + ) + cls.input_ids = get_input_ids(cls.model, num_samples=18) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/radix_cache/test_unified_radix_cache_kl_swa.py b/test/registered/radix_cache/test_unified_radix_cache_kl_swa.py new file mode 100644 index 000000000..1f8b95239 --- /dev/null +++ b/test/registered/radix_cache/test_unified_radix_cache_kl_swa.py @@ -0,0 +1,56 @@ +import unittest + +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.unified_radix_cache_kit import UnifiedRadixTreeTestMixin +from sglang.test.kl_multiturn_utils import get_input_ids +from sglang.test.test_utils import ( + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + is_in_ci, + popen_launch_server, +) + +register_cuda_ci(est_time=250, stage="base-c", runner_config="4-gpu-h100") + +SWA_MODEL = "openai/gpt-oss-20b" + + +class TestUnifiedSWARadixCache(UnifiedRadixTreeTestMixin, CustomTestCase): + """SWA hybrid + UnifiedRadixCache.""" + + kl_threshold = 0.03 + gsm8k_threshold = 0.7 + mmlu_threshold = 0.7 + + @unittest.skipIf(is_in_ci(), "SWA model mmlu eval not stable enough") + def test_mmlu(self): + super().test_mmlu() + + @classmethod + def setUpClass(cls): + cls.model = SWA_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=[ + "--tp-size", + "4", + "--mem-fraction-static", + "0.7", + "--disable-piecewise-cuda-graph", + ], + env={"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1"}, + ) + cls.input_ids = get_input_ids(cls.model, num_samples=18) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + +if __name__ == "__main__": + unittest.main()