[CI] Lower GSM8K baselines for B200 nightly after eval unification (#22136)
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2b4eeb8343
commit
18359aadc8
@@ -30,6 +30,7 @@ class AccuracyTestParams:
|
||||
top_p: Optional[float] = None
|
||||
top_k: Optional[int] = None
|
||||
repeat: Optional[int] = None
|
||||
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -87,6 +88,7 @@ def _run_simple_eval(
|
||||
top_p: Optional[float] = None,
|
||||
top_k: Optional[int] = None,
|
||||
repeat: Optional[int] = None,
|
||||
api: Optional[str] = None,
|
||||
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
||||
"""Run evaluation using simple_eval backend (run_eval.py).
|
||||
|
||||
@@ -111,6 +113,9 @@ def _run_simple_eval(
|
||||
num_threads=num_threads or 1024,
|
||||
)
|
||||
|
||||
if api is not None:
|
||||
args.api = api
|
||||
|
||||
if max_tokens is not None:
|
||||
args.max_tokens = max_tokens
|
||||
|
||||
@@ -489,6 +494,7 @@ def run_accuracy_test(
|
||||
top_p=params.top_p,
|
||||
top_k=params.top_k,
|
||||
repeat=params.repeat,
|
||||
api=params.api,
|
||||
)
|
||||
|
||||
if not success:
|
||||
|
||||
@@ -59,7 +59,7 @@ class FlashinferTrtllmGenMoeBackendFP8Base:
|
||||
)
|
||||
metrics = run_eval(args)
|
||||
print(f"{metrics=}")
|
||||
self.assertGreater(metrics["score"], 0.93)
|
||||
self.assertGreater(metrics["score"], 0.89)
|
||||
|
||||
|
||||
class FlashinferTrtllmGenMoeBackendBF16Base:
|
||||
|
||||
@@ -63,7 +63,10 @@ class TestDeepseekR1FP4Unified(unittest.TestCase):
|
||||
models=variants,
|
||||
test_name="DeepSeek-V3-0324-FP4 Unified",
|
||||
accuracy_params=AccuracyTestParams(
|
||||
dataset="gsm8k", baseline_accuracy=0.935
|
||||
dataset="gsm8k",
|
||||
baseline_accuracy=0.935,
|
||||
num_examples=200,
|
||||
api="completion",
|
||||
),
|
||||
performance_params=PerformanceTestParams(
|
||||
profile_dir="performance_profiles_deepseek_v3_fp4",
|
||||
|
||||
Reference in New Issue
Block a user