minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

Unify GSM8K eval path to Chat API for regression CI readiness (#21667)

Browse Source
This commit is contained in:
Liangsheng Yin
2026-04-01 17:12:19 -07:00
committed by GitHub
parent 1081a25983
commit d7256eb69a
79 changed files with 1349 additions and 1359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/quant/test_deepseek_v32_fp4_4gpu.py
+21 -19
View File
@@ -3,7 +3,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
@@ -56,23 +56,24 @@ class TestDeepseekV32FP4DP(CustomTestCase):
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
data_path=None,
num_questions=500,
parallel=500,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
@@ -123,23 +124,24 @@ class TestDeepseekV32FP4TP(CustomTestCase):
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
data_path=None,
num_questions=500,
parallel=500,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
test/registered/quant/test_deepseek_v32_fp4_mtp_4gpu.py
+21 -19
View File
@@ -6,7 +6,7 @@ import requests
from sglang.srt.environ import envs
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
@@ -72,15 +72,16 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
data_path=None,
num_questions=500,
parallel=500,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
@@ -92,10 +93,10 @@ class TestDeepseekV32FP4DPSpecV2(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 mtp)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
self.assertGreater(avg_spec_accept_length, 2.7)
def test_bs_1_speed(self):
@@ -162,15 +163,16 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
requests.get(self.base_url + "/flush_cache")
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=500,
num_threads=500,
num_shots=20,
data_path=None,
num_questions=500,
parallel=500,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
@@ -182,10 +184,10 @@ class TestDeepseekV32FP4TPSpecV2(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v32 mtp)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
self.assertGreater(avg_spec_accept_length, 2.7)
def test_bs_1_speed(self):
test/registered/quant/test_deepseek_v3_fp4_4gpu.py
+31 -28
View File
@@ -4,7 +4,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.send_one import BenchArgs, send_one_prompt
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
@@ -54,23 +54,24 @@ class TestDeepseekV3FP4(CustomTestCase):
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
data_path=None,
num_questions=1319,
parallel=1319,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
def test_bs_1_speed(self):
args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)
@@ -124,23 +125,24 @@ class TestDeepseekV3FP4CutlassMoE(CustomTestCase):
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
data_path=None,
num_questions=1319,
parallel=1319,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4-cutlass-moe)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
@@ -178,23 +180,24 @@ class TestDeepseekV3FP4SymmetricMemory(CustomTestCase):
self,
): # Append an "a" to make this test run first (alphabetically) to warm up the server
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=1319,
num_shots=8,
data_path=None,
num_questions=1319,
parallel=1319,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["accuracy"]=:.3f}\n'
f"### test_gsm8k (deepseek-v3-fp4)\n" f'{metrics["score"]=:.3f}\n'
)
self.assertGreater(metrics["accuracy"], 0.93)
self.assertGreater(metrics["score"], 0.93)
if __name__ == "__main__":
test/registered/quant/test_fp8_blockwise_gemm.py
+19 -17
View File
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -46,18 +46,19 @@ class FP8BlockwiseGemmBase:
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=200,
num_shots=8,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=200,
host=f"{parsed_url.scheme}://{parsed_url.hostname}",
port=parsed_url.port,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.8)
self.assertGreaterEqual(metrics["score"], 0.8)
class MXFP8GemmBase:
@@ -88,18 +89,19 @@ class MXFP8GemmBase:
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=200,
num_shots=8,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=200,
host=f"{parsed_url.scheme}://{parsed_url.hostname}",
port=parsed_url.port,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(metrics)
self.assertGreaterEqual(metrics["accuracy"], 0.8)
self.assertGreaterEqual(metrics["score"], 0.8)
class TestFP8BlockwiseGemmTriton(FP8BlockwiseGemmBase, unittest.TestCase):
test/registered/quant/test_fp8kv_triton.py
+9 -9
View File
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -41,17 +41,17 @@ class TestFP8KVCacheTritonBackend(CustomTestCase):
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=200,
host=f"{parsed_url.scheme}://{parsed_url.hostname}",
port=parsed_url.port,
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=200,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.70)
self.assertGreater(metrics["score"], 0.70)
if __name__ == "__main__":
test/registered/quant/test_int4fp8_moe.py
+9 -8
View File
@@ -2,7 +2,7 @@ from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_URL_FOR_TEST,
CustomTestCase,
@@ -45,14 +45,15 @@ class TestMixtralAccuracy(CustomTestCase):
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1400,
num_threads=128,
num_shots=8,
data_path=None,
num_questions=1400,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.56)
self.assertGreater(metrics["score"], 0.56)
test/registered/quant/test_modelopt_fp8.py
+9 -9
View File
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -35,17 +35,17 @@ class TestModeloptFP8(CustomTestCase):
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=200,
host=f"{parsed_url.scheme}://{parsed_url.hostname}",
port=parsed_url.port,
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=200,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["accuracy"], 0.70)
self.assertGreater(metrics["score"], 0.70)
if __name__ == "__main__":
test/registered/quant/test_nvfp4_gemm.py
+10 -10
View File
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
from sglang.srt.utils import get_device_sm, kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -47,18 +47,18 @@ class FP4GemmBase:
def test_gsm8k(self):
parsed_url = urlparse(self.base_url)
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1319,
max_new_tokens=512,
parallel=200,
host=f"{parsed_url.scheme}://{parsed_url.hostname}",
port=parsed_url.port,
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1319,
num_threads=200,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["accuracy"], 0.64)
self.assertGreater(metrics["score"], 0.64)
@unittest.skipIf(get_device_sm() < 100, "Test requires CUDA SM 100 or higher")
test/registered/quant/test_w4a8_deepseek_v3.py
+38 -38
View File
@@ -6,7 +6,7 @@ import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_DEEPSEEK_W4AFP8_MODEL_FOR_TEST,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
@@ -41,18 +41,18 @@ class TestDeepseekV3W4afp8(CustomTestCase):
def test_gsm8k(self):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=1200,
parallel=1200,
max_new_tokens=512,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=1200,
num_threads=1200,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"Eval accuracy of GSM8K: {metrics=}")
self.assertGreater(metrics["accuracy"], 0.92)
self.assertGreater(metrics["score"], 0.92)
class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
@@ -95,15 +95,15 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
self,
):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"{metrics=}")
server_info = requests.get(self.base_url + "/get_server_info")
@@ -115,10 +115,10 @@ class TestDeepseekV3W4Afp8Mtp(CustomTestCase):
if is_in_ci():
write_github_step_summary(
f"### test_gsm8k (deepseek-v3 mtp)\n"
f'{metrics["accuracy"]=:.3f}\n'
f'{metrics["score"]=:.3f}\n'
f"{avg_spec_accept_length=:.2f}\n"
)
self.assertGreater(metrics["accuracy"], 0.935)
self.assertGreater(metrics["score"], 0.935)
self.assertGreater(avg_spec_accept_length, 2.9)
@@ -163,18 +163,18 @@ class TestDeepseekV3W4Afp8DeepepNormal(CustomTestCase):
self,
):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"Eval accuracy of GSM8K: {metrics=}")
self.assertGreater(metrics["accuracy"], 0.92)
self.assertGreater(metrics["score"], 0.92)
class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase):
@@ -231,18 +231,18 @@ class TestDeepseekV3W4Afp8DeepepAutoMtp(CustomTestCase):
self,
):
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval_few_shot_gsm8k(args)
metrics = run_eval(args)
print(f"Eval accuracy of GSM8K: {metrics=}")
self.assertGreater(metrics["accuracy"], 0.92)
self.assertGreater(metrics["score"], 0.92)
if __name__ == "__main__":
test/registered/quant/test_w8a8_quantization.py
+9 -9
View File
@@ -6,7 +6,7 @@ import requests
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.few_shot_gsm8k import run_eval
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
@@ -51,17 +51,17 @@ class BaseW8A8Test(CustomTestCase):
self.skipTest("gsm8k_accuracy_threshold not set for this test")
args = SimpleNamespace(
num_shots=5,
data_path=None,
num_questions=200,
max_new_tokens=512,
parallel=128,
host="http://127.0.0.1",
port=int(self.base_url.split(":")[-1]),
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(metrics)
self.assertGreater(metrics["accuracy"], self.gsm8k_accuracy_threshold)
self.assertGreater(metrics["score"], self.gsm8k_accuracy_threshold)
def run_decode(self, max_new_tokens):
response = requests.post(