Unify GSM8K eval path to Chat API for regression CI readiness (#21667)
This commit is contained in:
@@ -13,7 +13,7 @@ from urllib.parse import urlparse
|
||||
import requests
|
||||
|
||||
from sglang.srt.utils import kill_process_tree
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.test_utils import (
|
||||
DEFAULT_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
|
||||
@@ -189,24 +189,24 @@ class TestPPWithHiCache(unittest.TestCase):
|
||||
|
||||
def test_eval_accuracy(self):
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=40,
|
||||
max_new_tokens=256,
|
||||
parallel=24,
|
||||
host=f"http://{self.base_host}",
|
||||
port=int(self.base_port),
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="gsm8k",
|
||||
api="completion",
|
||||
max_tokens=512,
|
||||
num_examples=40,
|
||||
num_threads=24,
|
||||
)
|
||||
|
||||
metrics_initial = run_eval_few_shot_gsm8k(args)
|
||||
self.assertGreater(metrics_initial["accuracy"], 0.6)
|
||||
metrics_initial = run_eval(args)
|
||||
self.assertGreater(metrics_initial["score"], 0.6)
|
||||
|
||||
self.flush_cache()
|
||||
|
||||
metrics_cached = run_eval_few_shot_gsm8k(args)
|
||||
self.assertGreater(metrics_cached["accuracy"], 0.6)
|
||||
metrics_cached = run_eval(args)
|
||||
self.assertGreater(metrics_cached["score"], 0.6)
|
||||
|
||||
accuracy_diff = abs(metrics_initial["accuracy"] - metrics_cached["accuracy"])
|
||||
accuracy_diff = abs(metrics_initial["score"] - metrics_cached["score"])
|
||||
self.assertLess(accuracy_diff, 0.05)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user