From 6d53615699888a847351f47b047f0b7187a81871 Mon Sep 17 00:00:00 2001 From: Khoa Pham Date: Tue, 2 Jun 2026 22:57:12 -0700 Subject: [PATCH] [Gemma4] Use hard GSM8K accuracy floor for 31B MTP test (#27101) Co-authored-by: Claude Opus 4.8 (1M context) --- test/registered/spec/test_gemma4_mtp_31b_extra.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/registered/spec/test_gemma4_mtp_31b_extra.py b/test/registered/spec/test_gemma4_mtp_31b_extra.py index 54590621c..c359a379e 100644 --- a/test/registered/spec/test_gemma4_mtp_31b_extra.py +++ b/test/registered/spec/test_gemma4_mtp_31b_extra.py @@ -28,14 +28,10 @@ TOPKS = (1, 3) DRAFT_TOKENS_BY_TOPK = {1: 6, 3: 12} GSM8K_NUM_EXAMPLES = 200 GSM8K_NUM_THREADS = 128 -GSM8K_SCORE_MARGIN = 0.03 SERVER_LAUNCH_TIMEOUT = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3 -# Initial values are seeded from current Gemma4 GSM8K observations in the -# cookbook. Replace each top-k entry with exact MTP first-200-sample scores as -# CI calibration data becomes available. -OBSERVED_GSM8K_SCORES = {1: 0.805, 3: 0.805} -GSM8K_SCORE_THRESHOLD = min(OBSERVED_GSM8K_SCORES.values()) - GSM8K_SCORE_MARGIN +# Hard accuracy floor for the MTP first-200-sample GSM8K run. +GSM8K_SCORE_THRESHOLD = 0.75 ACCEPT_LENGTH_THRESHOLD = 1.5