minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

[Gemma4] Use hard GSM8K accuracy floor for 31B MTP test (#27101)

Browse Source 
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Khoa Pham
2026-06-03 01:57:12 -04:00
committed by GitHub
co-authored by Claude Opus 4.8
parent b8d7351a74
commit 6d53615699
1 changed files with 2 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/registered/spec/test_gemma4_mtp_31b_extra.py
+2 -6
View File
@@ -28,14 +28,10 @@ TOPKS = (1, 3)
DRAFT_TOKENS_BY_TOPK = {1: 6, 3: 12}
GSM8K_NUM_EXAMPLES = 200
GSM8K_NUM_THREADS = 128
GSM8K_SCORE_MARGIN = 0.03
SERVER_LAUNCH_TIMEOUT = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3
# Initial values are seeded from current Gemma4 GSM8K observations in the
# cookbook. Replace each top-k entry with exact MTP first-200-sample scores as
# CI calibration data becomes available.
OBSERVED_GSM8K_SCORES = {1: 0.805, 3: 0.805}
GSM8K_SCORE_THRESHOLD = min(OBSERVED_GSM8K_SCORES.values()) - GSM8K_SCORE_MARGIN
# Hard accuracy floor for the MTP first-200-sample GSM8K run.
GSM8K_SCORE_THRESHOLD = 0.75
ACCEPT_LENGTH_THRESHOLD = 1.5