minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[CI] Migrate mgsm_en eval to gsm8k to remove openaipublic dependency (#21931)

Browse Source 
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com>
This commit is contained in:
Douglas Yang
2026-04-07 16:29:20 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6 Kangyan-Zhou
parent 7546d04c81
commit f6fc39569a
7 changed files with 82 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/amd/accuracy/mi30x/test_gsm8k_eval_amd.py
+27 -26
View File
@@ -1,7 +1,7 @@
"""
AMD GSM8K Evaluation Test (Migrated from test/srt/nightly/)
This test evaluates instruction-tuned models on the mgsm_en benchmark using chat completions.
This test evaluates instruction-tuned models on the gsm8k benchmark using chat completions.
Models are tested with various TP configurations on AMD GPUs.
Registry: nightly-amd suite (2-GPU tests)
@@ -35,34 +35,35 @@ from sglang.test.test_utils import (
register_amd_ci(est_time=3600, suite="nightly-amd", nightly=True)
MODEL_SCORE_THRESHOLDS = {
# Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores
# Llama 3.1 series
"meta-llama/Llama-3.1-8B-Instruct": 0.82,
"meta-llama/Llama-3.1-70B-Instruct": 0.95,
"meta-llama/Llama-3.1-8B-Instruct": 0.80, # 84.5% - 5%
"meta-llama/Llama-3.1-70B-Instruct": 0.89, # 94.1% - 5%
# Llama 3.2 series (smaller models)
"meta-llama/Llama-3.2-3B-Instruct": 0.55,
"meta-llama/Llama-3.2-3B-Instruct": 0.43, # 48.2% - 5%
# Mistral series
"mistralai/Mistral-7B-Instruct-v0.3": 0.55,
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.58,
"mistralai/Mistral-7B-Instruct-v0.3": 0.47, # 52.1% - 5%
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69, # 74.4% - 5% (lower if AMD scores differently)
# DeepSeek series
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81, # 86.4% - 5%
# Qwen2 series
"Qwen/Qwen2-57B-A14B-Instruct": 0.86,
"Qwen/Qwen2.5-7B-Instruct": 0.85,
"Qwen/Qwen2-57B-A14B-Instruct": 0.76, # 80.7% - 5% (official A14B score; 88.2% was the 72B)
"Qwen/Qwen2.5-7B-Instruct": 0.82, # 86.3% - 5%
# Qwen3 series
"Qwen/Qwen3-30B-A3B-Thinking-2507": 0.84, # MoE model verified on MI300X
"Qwen/Qwen3-8B": 0.77,
"Qwen/Qwen3-30B-A3B-Thinking-2507": 0.86, # 91.4% - 5% (full attention mode; ensure sufficient max_tokens)
"Qwen/Qwen3-8B": 0.76, # ~81% - 5%
# Google Gemma
"google/gemma-2-27b-it": 0.91,
"google/gemma-2-9b-it": 0.72,
"google/gemma-2-27b-it": 0.86, # 90.7% - 5%
"google/gemma-2-9b-it": 0.74, # 78.5% - 5%
# "neuralmagic/gemma-2-2b-it-FP8": 0.4, # Small 2B model - OOM on single GPU
# FP8 quantized models
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.8,
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.92,
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.81,
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.57,
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84,
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80, # 84.5% - 5%
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.46, # ~51% - 5%
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.1% - 5%
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.86, # 91.1% - 5%
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76, # 80.7% - 5% (official A14B score)
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69, # 74.4% - 5%
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81, # 86.4% - 5%
}
failing_models = {
@@ -185,7 +186,7 @@ def check_model_scores(results):
summary += line
print(f"\n{'='*60}")
print("SUMMARY - TP=2 Instruction Models (mgsm_en)")
print("SUMMARY - TP=2 Instruction Models (gsm8k)")
print(f"{'='*60}")
print(summary)
print(f"\n📊 Final Statistics:")
@@ -200,7 +201,7 @@ def check_model_scores(results):
raise AssertionError(f"The following models failed:\n{failure_msg}")
# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry
# Do not use `CustomTestCase` since `test_gsm8k_all_models` does not want retry
class TestNightlyGsm8KEval(unittest.TestCase):
@classmethod
def setUpClass(cls):
@@ -215,7 +216,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
]
cls.base_url = DEFAULT_URL_FOR_TEST
def test_mgsm_en_all_models(self):
def test_gsm8k_all_models(self):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
@@ -226,7 +227,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
print(f"\n{'='*60}")
print("AMD GSM8K Evaluation Test (TP=2 Instruction Models)")
print(f"{'='*60}")
print(f"Benchmark: mgsm_en (chat completions)")
print(f"Benchmark: gsm8k (chat completions)")
print(f"{'='*60}\n")
for model_group, is_fp8, is_tp2 in self.model_groups:
@@ -261,13 +262,13 @@ class TestNightlyGsm8KEval(unittest.TestCase):
args = SimpleNamespace(
base_url=self.base_url,
model=model,
eval_name="mgsm_en",
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
# Run eval with timing and retries
print(f"📊 Running mgsm_en evaluation...")
print(f"📊 Running gsm8k evaluation...")
eval_start = time.time()
threshold = MODEL_SCORE_THRESHOLDS.get(model)
metrics = None
test/registered/distributed/test_dp_attention_large.py
+2 -2
View File
@@ -56,11 +56,11 @@ class TestDPAttentionDP2TP4(
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_en(self):
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
test/registered/distributed/test_pp_single_node.py
+2 -2
View File
@@ -128,11 +128,11 @@ class TestDPAttentionDP2PP2(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_en(self):
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
test/registered/eval/test_text_models_gsm8k_eval.py
+22 -21
View File
@@ -26,28 +26,29 @@ NIGHTLY_EVAL_SERVER_TIMEOUT = 1800
register_cuda_ci(est_time=3600, suite="nightly-eval-text-2-gpu", nightly=True)
MODEL_SCORE_THRESHOLDS = {
"meta-llama/Llama-3.1-8B-Instruct": 0.82,
"mistralai/Mistral-7B-Instruct-v0.3": 0.58,
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
"google/gemma-2-27b-it": 0.91,
"meta-llama/Llama-3.1-70B-Instruct": 0.95,
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.616,
"Qwen/Qwen2-57B-A14B-Instruct": 0.86,
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83,
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.835,
"zai-org/GLM-4.5-Air-FP8": 0.75,
# The threshold of neuralmagic/gemma-2-2b-it-FP8 should be 0.6, but this model has some accuracy regression.
# The fix is tracked at https://github.com/sgl-project/sglang/issues/4324, we set it to 0.50, for now, to make CI green.
"neuralmagic/gemma-2-2b-it-FP8": 0.50,
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65,
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82,
# Thresholds set at 5% below reported GSM8K (5-shot/CoT) scores
"meta-llama/Llama-3.1-8B-Instruct": 0.80, # 84.5% - 5%
"mistralai/Mistral-7B-Instruct-v0.3": 0.47, # 52.1% - 5%
"deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.81, # 86.4% - 5%
"google/gemma-2-27b-it": 0.86, # 90.7% - 5%
"meta-llama/Llama-3.1-70B-Instruct": 0.89, # 94.1% - 5%
"mistralai/Mixtral-8x7B-Instruct-v0.1": 0.69, # 74.4% - 5%
"Qwen/Qwen2-57B-A14B-Instruct": 0.76, # 80.7% - 5% (official A14B score; 88.2% was the 72B)
"neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.80, # 84.5% - 5%
"neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.47, # 52.1% - 5%
"neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.81, # 86.4% - 5%
"zai-org/GLM-4.5-Air-FP8": 0.80, # ~85% - 5%
# GSM8K baseline for gemma-2-2b is ~40-45%; threshold set at 5% below.
# (Previously 0.50 based on MGSM-EN; tracked regression: https://github.com/sgl-project/sglang/issues/4324)
"neuralmagic/gemma-2-2b-it-FP8": 0.38, # ~43% - 5%
"neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.89, # 94.1% - 5%
"neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.69, # 74.4% - 5%
"neuralmagic/Qwen2-72B-Instruct-FP8": 0.86, # 91.1% - 5%
"neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.76, # 80.7% - 5% (official A14B score)
}
# Do not use `CustomTestCase` since `test_mgsm_en_all_models` does not want retry
# Do not use `CustomTestCase` since `test_gsm8k_all_models` does not want retry
class TestNightlyGsm8KEval(unittest.TestCase):
@classmethod
def setUpClass(cls):
@@ -66,7 +67,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
cls.base_url = DEFAULT_URL_FOR_TEST
def test_mgsm_en_all_models(self):
def test_gsm8k_all_models(self):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
@@ -91,7 +92,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
args = SimpleNamespace(
base_url=self.base_url,
model=model_setup.model_path,
eval_name="mgsm_en",
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
test/registered/piecewise_cuda_graph/test_piecewise_cuda_graph_support_1_gpu.py
+16 -16
View File
@@ -41,21 +41,19 @@ class TestPiecewiseCudaGraphQwen25VL(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_accuracy(self):
num_examples = 2000
def test_gsm8k_accuracy(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=num_examples,
num_threads=min(num_examples, 1024),
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
print(f"GSM8K Accuracy: {metrics['score']:.3f}")
self.assertGreaterEqual(metrics["score"], 0.70)
self.assertGreaterEqual(metrics["score"], 0.82)
class TestPiecewiseCudaGraphInternVL25(CustomTestCase):
@@ -79,21 +77,23 @@ class TestPiecewiseCudaGraphInternVL25(CustomTestCase):
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mgsm_accuracy(self):
num_examples = 2000
def test_gsm8k_accuracy(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mgsm_en",
num_examples=num_examples,
num_threads=min(num_examples, 1024),
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
print(f"MGSM Accuracy: {metrics['score']:.3f}")
print(f"GSM8K Accuracy: {metrics['score']:.3f}")
self.assertGreaterEqual(metrics["score"], 0.70)
# Baseline (no piecewise CUDA graph): 0.571 — this eval uses 5-shot
# concatenated text via chat API, which scores lower than reported
# benchmarks (~77.8%) that use proper CoT chat format. The threshold
# is set 5% below observed to catch catastrophic regressions.
self.assertGreaterEqual(metrics["score"], 0.54)
class TestPiecewiseCudaGraphQwen25VLEmbedding(CustomTestCase):
test/registered/quant/test_quantization.py
+8 -5
View File
@@ -19,9 +19,12 @@ from sglang.test.test_utils import (
register_cuda_ci(est_time=370, suite="stage-b-test-1-gpu-large")
MODEL_SCORE_THRESHOLDS = {
"hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.825,
"hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.825,
"hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.615,
# Baselines observed with gsm8k 5-shot concatenated format via chat API,
# which scores lower than reported benchmarks using proper CoT format.
# Thresholds set 5% below observed to catch catastrophic regressions.
"hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.74, # observed: 0.781
"hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.74, # observed: 0.785
"hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.36, # observed: 0.380
}
@@ -93,7 +96,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
]
cls.base_url = DEFAULT_URL_FOR_TEST
def test_mgsm_en_all_models(self):
def test_gsm8k_all_models(self):
warnings.filterwarnings(
"ignore", category=ResourceWarning, message="unclosed.*socket"
)
@@ -110,7 +113,7 @@ class TestNightlyGsm8KEval(unittest.TestCase):
args = SimpleNamespace(
base_url=self.base_url,
model=model,
eval_name="mgsm_en",
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
test/registered/scheduler/test_prefill_delayer.py
+5 -5
View File
@@ -428,10 +428,10 @@ class TestPrefillDelayerTokenUsageLowWatermark(CustomTestCase):
class TestPrefillDelayerAccuracy(CustomTestCase):
def test_1_mgsm_en_has_prefill_delayer(self):
def test_1_gsm8k_has_prefill_delayer(self):
self._run_accuracy_test(prefill_delayer=True)
def test_2_mgsm_en_no_prefill_delayer(self):
def test_2_gsm8k_no_prefill_delayer(self):
self._run_accuracy_test(prefill_delayer=False)
def _run_accuracy_test(self, prefill_delayer: bool):
@@ -454,14 +454,14 @@ class TestPrefillDelayerAccuracy(CustomTestCase):
args = SimpleNamespace(
base_url=base_url,
model=model,
eval_name="mgsm_en",
eval_name="gsm8k",
num_examples=None,
num_threads=1024,
)
metrics = run_eval(args)
print(f"=== mgsm_en ({prefill_delayer=}) ===")
print(f"=== gsm8k ({prefill_delayer=}) ===")
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.87)
self.assertGreater(metrics["score"], 0.57)
finally:
kill_process_tree(process.pid)