diff --git a/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py b/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py index aa7813ee5..9a37ed6d5 100644 --- a/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py +++ b/test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py @@ -1,7 +1,7 @@ """ AMD GSM8K Evaluation Test (Migrated from test/srt/nightly/) -This test evaluates instruction-tuned models on the mgsm_en benchmark using chat completions. +This test evaluates instruction-tuned models on the gsm8k benchmark using chat completions. Models are tested with various TP configurations on AMD GPUs. Registry: nightly-amd suite (2-GPU tests) @@ -35,34 +35,35 @@ from sglang.test.test_utils import ( register_amd_ci(est_time=3600, suite="nightly-amd", nightly=True) MODEL_SCORE_THRESHOLDS = { + # Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores # Llama 3.1 series - "meta-llama/Llama-3.1-8B-Instruct": 0.82, - "meta-llama/Llama-3.1-70B-Instruct": 0.95, + "meta-llama/Llama-3.1-8B-Instruct": 0.80, # 84.5% - 5% + "meta-llama/Llama-3.1-70B-Instruct": 0.89, # 94.1% - 5% # Llama 3.2 series (smaller models) - "meta-llama/Llama-3.2-3B-Instruct": 0.55, + "meta-llama/Llama-3.2-3B-Instruct": 0.43, # 48.2% - 5% # Mistral series - "mistralai/Mistral-7B-Instruct-v0.3": 0.55, - "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.58, + "mistralai/Mistral-7B-Instruct-v0.3": 0.47, # 52.1% - 5% + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69, # 74.4% - 5% (lower if AMD scores differently) # DeepSeek series - "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85, + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81, # 86.4% - 5% # Qwen2 series - "Qwen/Qwen2-57B-A14B-Instruct": 0.86, - "Qwen/Qwen2.5-7B-Instruct": 0.85, + "Qwen/Qwen2-57B-A14B-Instruct": 0.76, # 80.7% - 5% (official A14B score; 88.2% was the 72B) + "Qwen/Qwen2.5-7B-Instruct": 0.82, # 86.3% - 5% # Qwen3 series - "Qwen/Qwen3-30B-A3B-Thinking-2507": 0.84, # MoE model verified on MI300X - "Qwen/Qwen3-8B": 0.77, + "Qwen/Qwen3-30B-A3B-Thinking-2507": 0.86, # 91.4% - 5% (full attention mode; ensure sufficient max_tokens) + "Qwen/Qwen3-8B": 0.76, # ~81% - 5% # Google Gemma - "google/gemma-2-27b-it": 0.91, - "google/gemma-2-9b-it": 0.72, + "google/gemma-2-27b-it": 0.86, # 90.7% - 5% + "google/gemma-2-9b-it": 0.74, # 78.5% - 5% # "neuralmagic/gemma-2-2b-it-FP8": 0.4, # Small 2B model - OOM on single GPU # FP8 quantized models - "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.8, - "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, - "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, - "neuralmagic/Qwen2-72B-Instruct-FP8": 0.92, - "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.81, - "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.57, - "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84, + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80, # 84.5% - 5% + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.46, # ~51% - 5% + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.1% - 5% + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.86, # 91.1% - 5% + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76, # 80.7% - 5% (official A14B score) + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69, # 74.4% - 5% + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81, # 86.4% - 5% } failing_models = { @@ -185,7 +186,7 @@ def check_model_scores(results): summary += line print(f"\n{'='*60}") - print("SUMMARY - TP=2 Instruction Models (mgsm_en)") + print("SUMMARY - TP=2 Instruction Models (gsm8k)") print(f"{'='*60}") print(summary) print(f"\nšŸ“Š Final Statistics:") @@ -200,7 +201,7 @@ def check_model_scores(results): raise AssertionError(f"The following models failed:\n{failure_msg}") -# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry +# Do not use `CustomTestCase` since `test_gsm8k_all_models` does not want retry class TestNightlyGsm8KEval(unittest.TestCase): @classmethod def setUpClass(cls): @@ -215,7 +216,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): ] cls.base_url = DEFAULT_URL_FOR_TEST - def test_mgsm_en_all_models(self): + def test_gsm8k_all_models(self): warnings.filterwarnings( "ignore", category=ResourceWarning, message="unclosed.*socket" ) @@ -226,7 +227,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): print(f"\n{'='*60}") print("AMD GSM8K Evaluation Test (TP=2 Instruction Models)") print(f"{'='*60}") - print(f"Benchmark: mgsm_en (chat completions)") + print(f"Benchmark: gsm8k (chat completions)") print(f"{'='*60}\n") for model_group, is_fp8, is_tp2 in self.model_groups: @@ -261,13 +262,13 @@ class TestNightlyGsm8KEval(unittest.TestCase): args = SimpleNamespace( base_url=self.base_url, model=model, - eval_name="mgsm_en", + eval_name="gsm8k", num_examples=None, num_threads=1024, ) # Run eval with timing and retries - print(f"šŸ“Š Running mgsm_en evaluation...") + print(f"šŸ“Š Running gsm8k evaluation...") eval_start = time.time() threshold = MODEL_SCORE_THRESHOLDS.get(model) metrics = None diff --git a/test/registered/distributed/test_dp_attention_large.py b/test/registered/distributed/test_dp_attention_large.py index 48cdee862..3e1d65f74 100644 --- a/test/registered/distributed/test_dp_attention_large.py +++ b/test/registered/distributed/test_dp_attention_large.py @@ -56,11 +56,11 @@ class TestDPAttentionDP2TP4( def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mgsm_en(self): + def test_gsm8k(self): args = SimpleNamespace( base_url=self.base_url, model=self.model, - eval_name="mgsm_en", + eval_name="gsm8k", num_examples=None, num_threads=1024, ) diff --git a/test/registered/distributed/test_pp_single_node.py b/test/registered/distributed/test_pp_single_node.py index 76e1c068d..0dd5d4fe8 100644 --- a/test/registered/distributed/test_pp_single_node.py +++ b/test/registered/distributed/test_pp_single_node.py @@ -128,11 +128,11 @@ class TestDPAttentionDP2PP2(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mgsm_en(self): + def test_gsm8k(self): args = SimpleNamespace( base_url=self.base_url, model=self.model, - eval_name="mgsm_en", + eval_name="gsm8k", num_examples=None, num_threads=1024, ) diff --git a/test/registered/eval/test_text_models_gsm8k_eval.py b/test/registered/eval/test_text_models_gsm8k_eval.py index 943689542..c2974439c 100644 --- a/test/registered/eval/test_text_models_gsm8k_eval.py +++ b/test/registered/eval/test_text_models_gsm8k_eval.py @@ -26,28 +26,29 @@ NIGHTLY_EVAL_SERVER_TIMEOUT = 1800 register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True) MODEL_SCORE_THRESHOLDS = { - "meta-llama/Llama-3.1-8B-Instruct": 0.82, - "mistralai/Mistral-7B-Instruct-v0.3": 0.58, - "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85, - "google/gemma-2-27b-it": 0.91, - "meta-llama/Llama-3.1-70B-Instruct": 0.95, - "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.616, - "Qwen/Qwen2-57B-A14B-Instruct": 0.86, - "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83, - "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54, - "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.835, - "zai-org/GLM-4.5-Air-FP8": 0.75, - # The threshold of neuralmagic/gemma-2-2b-it-FP8 should be 0.6, but this model has some accuracy regression. - # The fix is tracked at https://github.com/sgl-project/sglang/issues/4324, we set it to 0.50, for now, to make CI green. - "neuralmagic/gemma-2-2b-it-FP8": 0.50, - "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94, - "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65, - "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94, - "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82, + # Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores + "meta-llama/Llama-3.1-8B-Instruct": 0.80, # 84.5% - 5% + "mistralai/Mistral-7B-Instruct-v0.3": 0.47, # 52.1% - 5% + "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81, # 86.4% - 5% + "google/gemma-2-27b-it": 0.86, # 90.7% - 5% + "meta-llama/Llama-3.1-70B-Instruct": 0.89, # 94.1% - 5% + "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69, # 74.4% - 5% + "Qwen/Qwen2-57B-A14B-Instruct": 0.76, # 80.7% - 5% (official A14B score; 88.2% was the 72B) + "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80, # 84.5% - 5% + "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.47, # 52.1% - 5% + "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81, # 86.4% - 5% + "zai-org/GLM-4.5-Air-FP8": 0.80, # ~85% - 5% + # GSM8K baseline for gemma-2-2b is ~40-45%; threshold set at 5% below. + # (Previously 0.50 based on MGSM-EN; tracked regression: https://github.com/sgl-project/sglang/issues/4324) + "neuralmagic/gemma-2-2b-it-FP8": 0.38, # ~43% - 5% + "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.1% - 5% + "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69, # 74.4% - 5% + "neuralmagic/Qwen2-72B-Instruct-FP8": 0.86, # 91.1% - 5% + "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76, # 80.7% - 5% (official A14B score) } -# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry +# Do not use `CustomTestCase` since `test_gsm8k_all_models` does not want retry class TestNightlyGsm8KEval(unittest.TestCase): @classmethod def setUpClass(cls): @@ -66,7 +67,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): cls.base_url = DEFAULT_URL_FOR_TEST - def test_mgsm_en_all_models(self): + def test_gsm8k_all_models(self): warnings.filterwarnings( "ignore", category=ResourceWarning, message="unclosed.*socket" ) @@ -91,7 +92,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): args = SimpleNamespace( base_url=self.base_url, model=model_setup.model_path, - eval_name="mgsm_en", + eval_name="gsm8k", num_examples=None, num_threads=1024, ) diff --git a/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py b/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py index e38b59f5b..ce6fe2291 100644 --- a/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py +++ b/test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py @@ -41,21 +41,19 @@ class TestPiecewiseCudaGraphQwen25VL(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mgsm_accuracy(self): - num_examples = 2000 - + def test_gsm8k_accuracy(self): args = SimpleNamespace( base_url=self.base_url, model=self.model, - eval_name="mgsm_en", - num_examples=num_examples, - num_threads=min(num_examples, 1024), + eval_name="gsm8k", + num_examples=None, + num_threads=1024, ) metrics = run_eval(args) - print(f"MGSM Accuracy: {metrics['score']:.3f}") + print(f"GSM8K Accuracy: {metrics['score']:.3f}") - self.assertGreaterEqual(metrics["score"], 0.70) + self.assertGreaterEqual(metrics["score"], 0.82) class TestPiecewiseCudaGraphInternVL25(CustomTestCase): @@ -79,21 +77,23 @@ class TestPiecewiseCudaGraphInternVL25(CustomTestCase): def tearDownClass(cls): kill_process_tree(cls.process.pid) - def test_mgsm_accuracy(self): - num_examples = 2000 - + def test_gsm8k_accuracy(self): args = SimpleNamespace( base_url=self.base_url, model=self.model, - eval_name="mgsm_en", - num_examples=num_examples, - num_threads=min(num_examples, 1024), + eval_name="gsm8k", + num_examples=None, + num_threads=1024, ) metrics = run_eval(args) - print(f"MGSM Accuracy: {metrics['score']:.3f}") + print(f"GSM8K Accuracy: {metrics['score']:.3f}") - self.assertGreaterEqual(metrics["score"], 0.70) + # Baseline (no piecewise CUDA graph): 0.571 ā€” this eval uses 5-shot + # concatenated text via chat API, which scores lower than reported + # benchmarks (~77.8%) that use proper CoT chat format. The threshold + # is set 5% below observed to catch catastrophic regressions. + self.assertGreaterEqual(metrics["score"], 0.54) class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase): diff --git a/test/registered/quant/test_quantization.py b/test/registered/quant/test_quantization.py index cdb1f0970..cdf0b0e61 100644 --- a/test/registered/quant/test_quantization.py +++ b/test/registered/quant/test_quantization.py @@ -19,9 +19,12 @@ from sglang.test.test_utils import ( register_cuda_ci(est_time=370, suite="stage-b-test-1-gpu-large") MODEL_SCORE_THRESHOLDS = { - "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.825, - "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.825, - "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.615, + # Baselines observed with gsm8k 5-shot concatenated format via chat API, + # which scores lower than reported benchmarks using proper CoT format. + # Thresholds set 5% below observed to catch catastrophic regressions. + "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.74, # observed: 0.781 + "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.74, # observed: 0.785 + "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.36, # observed: 0.380 } @@ -93,7 +96,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): ] cls.base_url = DEFAULT_URL_FOR_TEST - def test_mgsm_en_all_models(self): + def test_gsm8k_all_models(self): warnings.filterwarnings( "ignore", category=ResourceWarning, message="unclosed.*socket" ) @@ -110,7 +113,7 @@ class TestNightlyGsm8KEval(unittest.TestCase): args = SimpleNamespace( base_url=self.base_url, model=model, - eval_name="mgsm_en", + eval_name="gsm8k", num_examples=None, num_threads=1024, ) diff --git a/test/registered/scheduler/test_prefill_delayer.py b/test/registered/scheduler/test_prefill_delayer.py index 66ea497bf..493346fda 100644 --- a/test/registered/scheduler/test_prefill_delayer.py +++ b/test/registered/scheduler/test_prefill_delayer.py @@ -428,10 +428,10 @@ class TestPrefillDelayerTokenUsageLowWatermark(CustomTestCase): class TestPrefillDelayerAccuracy(CustomTestCase): - def test_1_mgsm_en_has_prefill_delayer(self): + def test_1_gsm8k_has_prefill_delayer(self): self._run_accuracy_test(prefill_delayer=True) - def test_2_mgsm_en_no_prefill_delayer(self): + def test_2_gsm8k_no_prefill_delayer(self): self._run_accuracy_test(prefill_delayer=False) def _run_accuracy_test(self, prefill_delayer: bool): @@ -454,14 +454,14 @@ class TestPrefillDelayerAccuracy(CustomTestCase): args = SimpleNamespace( base_url=base_url, model=model, - eval_name="mgsm_en", + eval_name="gsm8k", num_examples=None, num_threads=1024, ) metrics = run_eval(args) - print(f"=== mgsm_en ({prefill_delayer=}) ===") + print(f"=== gsm8k ({prefill_delayer=}) ===") print(f"{metrics=}") - self.assertGreater(metrics["score"], 0.87) + self.assertGreater(metrics["score"], 0.57) finally: kill_process_tree(process.pid)