minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[CI] Split PP tests into base and extra suites (#26648)

Browse Source 
Signed-off-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Shangming Cai
2026-05-29 12:15:27 +08:00
committed by GitHub
parent 40f91e6697
commit fba083c80f
2 changed files with 359 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/pp/test_pp_single_node.py
+4 -329
View File
@@ -1,13 +1,12 @@
"""
Usage:
python3 -m unittest test_pp_single_node.TestPPAccuracy.test_gsm8k
python3 -m unittest test_pp_single_node.TestQwenPPAccuracy.test_pp_consistency
python3 -m unittest test_pp_single_node.TestFixedBugs.test_chunked_prefill_with_small_bs
python3 -m unittest test_pp_single_node.TestQwenVLPPAccuracy.test_mmmu
python3 -m unittest test_pp_single_node.TestDPAttentionDP2PP2.test_gsm8k
python3 -m unittest test_pp_single_node.TestGemma4PPAccuracy.test_gsm8k
python3 -m unittest test_pp_single_node.TestGemma4PPAccuracy.test_mmmu
python3 -m unittest test_pp_single_node.TestGemma4PLEPPAccuracy.test_gsm8k
python3 -m unittest test_pp_single_node.TestPPMixedChunk.test_gsm8k
python3 -m unittest test_pp_single_node.TestFixedBugs.test_chunked_prefill_with_small_bs
"""
import time
@@ -26,8 +25,6 @@ from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST,
DEFAULT_MODEL_NAME_FOR_TEST_GEMMA4_PLE_PP,
DEFAULT_MODEL_NAME_FOR_TEST_GEMMA4_PP,
DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP,
DEFAULT_MODEL_NAME_FOR_TEST_VL_PP,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
CustomTestCase,
@@ -37,8 +34,8 @@ from sglang.test.test_utils import (
run_bench_one_batch_server,
)
register_cuda_ci(est_time=554, stage="base-c", runner_config="4-gpu-h100")
register_amd_ci(est_time=650, suite="stage-c-test-4-gpu-amd")
register_cuda_ci(est_time=500, stage="base-c", runner_config="4-gpu-h100")
register_amd_ci(est_time=500, suite="stage-c-test-4-gpu-amd")
class TestPPAccuracy(unittest.TestCase):
@@ -148,65 +145,6 @@ class TestDPAttentionDP2PP2(CustomTestCase):
self.assertGreater(metrics["score"], 0.8)
@unittest.skipIf(
is_in_amd_ci(),
"VLM PP accuracy too low on AMD (0.48-0.50 with both aiter and triton)",
)
class TestQwenVLPPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_VL_PP
cls.base_url = "http://127.0.0.1:23333"
cls.process = popen_launch_server(
DEFAULT_MODEL_NAME_FOR_TEST_VL_PP,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
1,
"--pp-size",
4,
"--chunked-prefill-size",
8192,
"--enable-multimodal",
],
)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.65)
# Wait a little bit so that the memory check happens.
time.sleep(4)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
def test_mmmu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmmu",
num_examples=None,
num_threads=32,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.26)
@unittest.skipIf(
is_in_amd_ci(),
"Gemma4 PP not yet validated on AMD",
@@ -338,225 +276,6 @@ class TestGemma4PLEPPAccuracy(unittest.TestCase):
time.sleep(4)
class TestQwenPPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23334" # different ports to avoid conflicts
cls.model_name = "Qwen/Qwen3-8B" # replace with your Qwen Model if needed
def run_gsm8k_test(self, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(pp_size=1)
pp_metrics = self.run_gsm8k_test(pp_size=2)
print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.74)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.02,
msg=(
f"PP accuracy dropped more than 2% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
@unittest.skipIf(is_in_amd_ci(), "PP consistency too flaky on AMD 4-GPU runners")
class TestQwenPPTieWeightsAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23335" # different ports to avoid conflicts
cls.model_name = (
"Qwen/Qwen3-0.6B" # qwen3 < 8B all have tie_word_embeddings = True
)
def run_gsm8k_test(self, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(pp_size=1)
pp_metrics = self.run_gsm8k_test(pp_size=2)
print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.38)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.02,
msg=(
f"PP accuracy dropped more than 2% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
class TestQwenMoePPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23336" # different ports to avoid conflicts
cls.model_name = "Qwen/Qwen3-30B-A3B" # replace with your Qwen Model if needed
def run_gsm8k_test(self, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(pp_size=1)
pp_metrics = self.run_gsm8k_test(pp_size=2)
print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.74)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.02,
msg=(
f"PP accuracy dropped more than 2% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
@unittest.skipIf(
is_in_ci(), "Qwen35 PP consistency too flaky on H100 and AMD 4-GPU runners"
)
class TestQwen35PPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23337" # different ports to avoid conflicts
cls.model_name = (
"Qwen/Qwen3.5-35B-A3B" # replace with your Qwen Model if needed
)
def run_gsm8k_test(self, tp_size, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
tp_size,
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(tp_size=2, pp_size=1)
pp_metrics = self.run_gsm8k_test(tp_size=1, pp_size=2)
print(f"[Qwen35 PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.83)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.05,
msg=(
f"PP accuracy dropped more than 5% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
class TestPPMixedChunk(CustomTestCase):
@classmethod
def setUpClass(cls):
@@ -633,49 +352,5 @@ class TestFixedBugs(unittest.TestCase):
)
@unittest.skipIf(
is_in_ci(), "Skipping GLM41V PP accuracy test before it gets more stable"
)
class TestGLM41VPPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
1,
"--pp-size",
2,
"--chunked-prefill-size",
8192,
"--enable-multimodal",
"--reasoning-parser",
"glm45",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmmu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmmu",
num_examples=None,
num_threads=32,
response_answer_regex=r"<\|begin_of_box\|>(.*)<\|end_of_box\|>",
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.45)
if __name__ == "__main__":
unittest.main()
test/registered/pp/test_pp_single_node_extra.py
+355
View File
@@ -0,0 +1,355 @@
"""
Usage:
python3 -m unittest test_pp_single_node_extra.TestQwenVLPPAccuracy.test_gsm8k
python3 -m unittest test_pp_single_node_extra.TestQwenPPAccuracy.test_pp_consistency
python3 -m unittest test_pp_single_node_extra.TestQwenPPTieWeightsAccuracy.test_pp_consistency
python3 -m unittest test_pp_single_node_extra.TestQwenMoePPAccuracy.test_pp_consistency
python3 -m unittest test_pp_single_node_extra.TestQwen35PPAccuracy.test_pp_consistency
python3 -m unittest test_pp_single_node_extra.TestGLM41VPPAccuracy.test_mmmu
"""
import time
import unittest
from types import SimpleNamespace
from sglang.srt.utils import kill_process_tree
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP,
DEFAULT_MODEL_NAME_FOR_TEST_VL_PP,
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
DEFAULT_URL_FOR_TEST,
is_in_amd_ci,
is_in_ci,
popen_launch_server,
)
register_cuda_ci(est_time=350, stage="extra-b", runner_config="4-gpu-h100")
register_amd_ci(est_time=350, suite="stage-c-test-4-gpu-amd")
@unittest.skipIf(
is_in_amd_ci(),
"VLM PP accuracy too low on AMD (0.48-0.50 with both aiter and triton)",
)
class TestQwenVLPPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_VL_PP
cls.base_url = "http://127.0.0.1:23333"
cls.process = popen_launch_server(
DEFAULT_MODEL_NAME_FOR_TEST_VL_PP,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
1,
"--pp-size",
4,
"--chunked-prefill-size",
8192,
"--enable-multimodal",
],
)
def test_gsm8k(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=200,
num_threads=128,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreaterEqual(metrics["score"], 0.65)
# Wait a little bit so that the memory check happens.
time.sleep(4)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
def test_mmmu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmmu",
num_examples=None,
num_threads=32,
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.26)
class TestQwenPPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23334" # different ports to avoid conflicts
cls.model_name = "Qwen/Qwen3-8B" # replace with your Qwen Model if needed
def run_gsm8k_test(self, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
@unittest.skipIf(is_in_ci(), "To reduce the CI execution time.")
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(pp_size=1)
pp_metrics = self.run_gsm8k_test(pp_size=2)
print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.74)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.02,
msg=(
f"PP accuracy dropped more than 2% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
@unittest.skipIf(is_in_amd_ci(), "PP consistency too flaky on AMD 4-GPU runners")
class TestQwenPPTieWeightsAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23335" # different ports to avoid conflicts
cls.model_name = (
"Qwen/Qwen3-0.6B" # qwen3 < 8B all have tie_word_embeddings = True
)
def run_gsm8k_test(self, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(pp_size=1)
pp_metrics = self.run_gsm8k_test(pp_size=2)
print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.38)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.02,
msg=(
f"PP accuracy dropped more than 2% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
class TestQwenMoePPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23336" # different ports to avoid conflicts
cls.model_name = "Qwen/Qwen3-30B-A3B" # replace with your Qwen Model if needed
def run_gsm8k_test(self, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(pp_size=1)
pp_metrics = self.run_gsm8k_test(pp_size=2)
print(f"[Qwen PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.74)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.02,
msg=(
f"PP accuracy dropped more than 2% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
@unittest.skipIf(
is_in_ci(), "Qwen35 PP consistency too flaky on H100 and AMD 4-GPU runners"
)
class TestQwen35PPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.base_url = "http://127.0.0.1:23337" # different ports to avoid conflicts
cls.model_name = (
"Qwen/Qwen3.5-35B-A3B" # replace with your Qwen Model if needed
)
def run_gsm8k_test(self, tp_size, pp_size):
process = popen_launch_server(
self.model_name,
self.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
tp_size,
"--pp-size",
pp_size,
"--chunked-prefill-size",
256,
],
)
try:
args = SimpleNamespace(
base_url=self.base_url,
model=self.model_name,
eval_name="gsm8k",
api="completion",
max_tokens=512,
num_examples=512,
num_threads=128,
)
metrics = run_eval(args)
time.sleep(5)
return metrics
finally:
kill_process_tree(process.pid)
def test_pp_consistency(self):
baseline = self.run_gsm8k_test(tp_size=2, pp_size=1)
pp_metrics = self.run_gsm8k_test(tp_size=1, pp_size=2)
print(f"[Qwen35 PP Comparison] Baseline: {baseline} | PP: {pp_metrics}")
self.assertGreaterEqual(baseline["score"], 0.83)
self.assertGreaterEqual(
pp_metrics["score"],
baseline["score"] - 0.05,
msg=(
f"PP accuracy dropped more than 5% compared to baseline. "
f"Baseline: {baseline['score']:.2%}, PP: {pp_metrics['score']:.2%}"
),
)
@unittest.skipIf(
is_in_ci(), "Skipping GLM41V PP accuracy test before it gets more stable"
)
class TestGLM41VPPAccuracy(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.model = DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP
cls.base_url = DEFAULT_URL_FOR_TEST
cls.process = popen_launch_server(
DEFAULT_MODEL_NAME_FOR_TEST_GLM_41V_PP,
cls.base_url,
timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
other_args=[
"--tp-size",
1,
"--pp-size",
2,
"--chunked-prefill-size",
8192,
"--enable-multimodal",
"--reasoning-parser",
"glm45",
],
)
@classmethod
def tearDownClass(cls):
kill_process_tree(cls.process.pid)
def test_mmmu(self):
args = SimpleNamespace(
base_url=self.base_url,
model=self.model,
eval_name="mmmu",
num_examples=None,
num_threads=32,
response_answer_regex=r"<\|begin_of_box\|>(.*)<\|end_of_box\|>",
)
metrics = run_eval(args)
print(f"{metrics=}")
self.assertGreater(metrics["score"], 0.45)
if __name__ == "__main__":
unittest.main()