Unify GSM8K eval path to Chat API for regression CI readiness (#21667)
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
|
||||
from sglang.test.run_eval import run_eval
|
||||
from sglang.test.server_fixtures.disaggregation_fixture import (
|
||||
PDDisaggregationServerBase,
|
||||
)
|
||||
@@ -70,18 +70,18 @@ class TestDisaggregationPiecewiseCudaGraph(PDDisaggregationServerBase):
|
||||
def test_gsm8k_accuracy(self):
|
||||
"""Verify that piecewise cuda graph works correctly in prefill server"""
|
||||
args = SimpleNamespace(
|
||||
num_shots=5,
|
||||
data_path=None,
|
||||
num_questions=200,
|
||||
max_new_tokens=512,
|
||||
parallel=128,
|
||||
host=f"http://{self.base_host}",
|
||||
port=int(self.lb_port),
|
||||
base_url=self.base_url,
|
||||
model=self.model,
|
||||
eval_name="gsm8k",
|
||||
api="completion",
|
||||
max_tokens=512,
|
||||
num_examples=200,
|
||||
num_threads=128,
|
||||
)
|
||||
metrics = run_eval_few_shot_gsm8k(args)
|
||||
print(f"GSM8K accuracy with piecewise cuda graph: {metrics['accuracy']:.3f}")
|
||||
metrics = run_eval(args)
|
||||
print(f"GSM8K accuracy with piecewise cuda graph: {metrics['score']:.3f}")
|
||||
|
||||
self.assertGreater(metrics["accuracy"], 0.62)
|
||||
self.assertGreater(metrics["score"], 0.62)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user