[CI] Lower GSM8K baselines for B200 nightly after eval unification (#22136)
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2b4eeb8343
commit
18359aadc8
@@ -30,6 +30,7 @@ class AccuracyTestParams:
|
||||
top_p: Optional[float] = None
|
||||
top_k: Optional[int] = None
|
||||
repeat: Optional[int] = None
|
||||
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -87,6 +88,7 @@ def _run_simple_eval(
|
||||
top_p: Optional[float] = None,
|
||||
top_k: Optional[int] = None,
|
||||
repeat: Optional[int] = None,
|
||||
api: Optional[str] = None,
|
||||
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
||||
"""Run evaluation using simple_eval backend (run_eval.py).
|
||||
|
||||
@@ -111,6 +113,9 @@ def _run_simple_eval(
|
||||
num_threads=num_threads or 1024,
|
||||
)
|
||||
|
||||
if api is not None:
|
||||
args.api = api
|
||||
|
||||
if max_tokens is not None:
|
||||
args.max_tokens = max_tokens
|
||||
|
||||
@@ -489,6 +494,7 @@ def run_accuracy_test(
|
||||
top_p=params.top_p,
|
||||
top_k=params.top_k,
|
||||
repeat=params.repeat,
|
||||
api=params.api,
|
||||
)
|
||||
|
||||
if not success:
|
||||
|
||||
Reference in New Issue
Block a user