[CI] Lower GSM8K baselines for B200 nightly after eval unification (#22136)
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
2b4eeb8343
commit
18359aadc8
@@ -30,6 +30,7 @@ class AccuracyTestParams:
|
|||||||
top_p: Optional[float] = None
|
top_p: Optional[float] = None
|
||||||
top_k: Optional[int] = None
|
top_k: Optional[int] = None
|
||||||
repeat: Optional[int] = None
|
repeat: Optional[int] = None
|
||||||
|
api: Optional[str] = None # "chat" or "completion"; defaults to "chat" in run_eval
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -87,6 +88,7 @@ def _run_simple_eval(
|
|||||||
top_p: Optional[float] = None,
|
top_p: Optional[float] = None,
|
||||||
top_k: Optional[int] = None,
|
top_k: Optional[int] = None,
|
||||||
repeat: Optional[int] = None,
|
repeat: Optional[int] = None,
|
||||||
|
api: Optional[str] = None,
|
||||||
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
) -> Tuple[bool, Optional[str], Optional[dict]]:
|
||||||
"""Run evaluation using simple_eval backend (run_eval.py).
|
"""Run evaluation using simple_eval backend (run_eval.py).
|
||||||
|
|
||||||
@@ -111,6 +113,9 @@ def _run_simple_eval(
|
|||||||
num_threads=num_threads or 1024,
|
num_threads=num_threads or 1024,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if api is not None:
|
||||||
|
args.api = api
|
||||||
|
|
||||||
if max_tokens is not None:
|
if max_tokens is not None:
|
||||||
args.max_tokens = max_tokens
|
args.max_tokens = max_tokens
|
||||||
|
|
||||||
@@ -489,6 +494,7 @@ def run_accuracy_test(
|
|||||||
top_p=params.top_p,
|
top_p=params.top_p,
|
||||||
top_k=params.top_k,
|
top_k=params.top_k,
|
||||||
repeat=params.repeat,
|
repeat=params.repeat,
|
||||||
|
api=params.api,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not success:
|
if not success:
|
||||||
|
|||||||
@@ -59,7 +59,7 @@ class FlashinferTrtllmGenMoeBackendFP8Base:
|
|||||||
)
|
)
|
||||||
metrics = run_eval(args)
|
metrics = run_eval(args)
|
||||||
print(f"{metrics=}")
|
print(f"{metrics=}")
|
||||||
self.assertGreater(metrics["score"], 0.93)
|
self.assertGreater(metrics["score"], 0.89)
|
||||||
|
|
||||||
|
|
||||||
class FlashinferTrtllmGenMoeBackendBF16Base:
|
class FlashinferTrtllmGenMoeBackendBF16Base:
|
||||||
|
|||||||
@@ -63,7 +63,10 @@ class TestDeepseekR1FP4Unified(unittest.TestCase):
|
|||||||
models=variants,
|
models=variants,
|
||||||
test_name="DeepSeek-V3-0324-FP4 Unified",
|
test_name="DeepSeek-V3-0324-FP4 Unified",
|
||||||
accuracy_params=AccuracyTestParams(
|
accuracy_params=AccuracyTestParams(
|
||||||
dataset="gsm8k", baseline_accuracy=0.935
|
dataset="gsm8k",
|
||||||
|
baseline_accuracy=0.935,
|
||||||
|
num_examples=200,
|
||||||
|
api="completion",
|
||||||
),
|
),
|
||||||
performance_params=PerformanceTestParams(
|
performance_params=PerformanceTestParams(
|
||||||
profile_dir="performance_profiles_deepseek_v3_fp4",
|
profile_dir="performance_profiles_deepseek_v3_fp4",
|
||||||
|
|||||||
Reference in New Issue
Block a user