minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Unify GSM8K eval path to Chat API for regression CI readiness (#21667)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-04-01 17:12:19 -07:00
committed by GitHub
parent 1081a25983
commit d7256eb69a
79 changed files with 1349 additions and 1359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/models/test_compressed_tensors_models.py
+10 -10
View File
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -35,21 +35,21 @@ class TestCompressedTensorsLlama3FP8(CustomTestCase):
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
if is_hip():
# Lower threshold for AMD because FP8 dtype differs (fp8_fnuz)
self.assertGreaterEqual(metrics["accuracy"], 0.40)
self.assertGreaterEqual(metrics["score"], 0.40)
else:
self.assertGreaterEqual(metrics["accuracy"], 0.45)
self.assertGreaterEqual(metrics["score"], 0.45)
if __name__ == "__main__":
test/registered/models/test_kimi_linear_models.py
+9 -9
View File
@@ -3,7 +3,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -32,17 +32,17 @@ class TestKimiLinear(CustomTestCase):
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.88)
self.assertGreater(metrics["score"], 0.88)
if __name__ == "__main__":
test/registered/models/test_qwen_models.py
+17 -17
View File
@@ -5,7 +5,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -35,17 +35,17 @@ class TestQwen2(CustomTestCase):
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.78)
self.assertGreater(metrics["score"], 0.78)
class TestQwen2FP8(CustomTestCase):
@@ -66,17 +66,17 @@ class TestQwen2FP8(CustomTestCase):
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.78)
self.assertGreater(metrics["score"], 0.78)
if __name__ == "__main__":
test/registered/models/test_transformers_models.py
+9 -12
View File
@@ -10,6 +10,7 @@ import torch
from sglang.srt.utils import is_hip, kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.runners import DEFAULT_PROMPTS, SRTRunner, check_close_model_outputs
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
@@ -50,26 +51,22 @@ class TestTransformersFallbackEndpoint(CustomTestCase):
num_examples=64,
num_threads=32,
)
from sglang.test.run_eval import run_eval
metrics = run_eval(args)
self.assertGreaterEqual(metrics["score"], self.mmlu_lower_bound)
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
from sglang.test.few_shot_gsm8k import run_eval
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], self.gsm8k_lower_bound)
self.assertGreater(metrics["score"], self.gsm8k_lower_bound)
@unittest.skipIf(is_hip(), "TorchAO int4wo quantization is not supported on AMD GPUs")