diff --git a/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py b/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py index 1d58ed7ce..2a5c89bff 100644 --- a/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py +++ b/test/registered/spec/test_gemma4_mtp_26b_a4b_extra.py @@ -31,10 +31,11 @@ GSM8K_NUM_THREADS = 128 GSM8K_SCORE_MARGIN = 0.03 SERVER_LAUNCH_TIMEOUT = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH * 3 -# Initial values are seeded from current Gemma4 GSM8K observations in the -# cookbook. Replace each top-k entry with exact MTP first-200-sample scores as -# CI calibration data becomes available. -OBSERVED_GSM8K_SCORES = {1: 0.450, 3: 0.450} +# Calibrated from deterministic-inference GSM8K runs (200 examples, 5-shot, +# greedy, triton, TP=2). With --enable-deterministic-inference the per-topk +# score is reproducible run-to-run (std=0 over N=20): topk=1 -> 0.445, +# topk=3 -> 0.440. +OBSERVED_GSM8K_SCORES = {1: 0.445, 3: 0.440} GSM8K_SCORE_THRESHOLD = min(OBSERVED_GSM8K_SCORES.values()) - GSM8K_SCORE_MARGIN ACCEPT_LENGTH_THRESHOLD = 1.5 @@ -84,6 +85,9 @@ class TestGemma4MTP26BA4B(CustomTestCase): "--max-total-tokens", "32768", "--skip-server-warmup", + # Batch-invariant kernels make the GSM8K score reproducible + # run-to-run; without this the topk=3 score swings ~0.33-0.50. + "--enable-deterministic-inference", ] if TENSOR_PARALLEL_SIZE > 1: args += ["--tp-size", str(TENSOR_PARALLEL_SIZE)]