From 6f1412f4f58db045acb80d9477251075bf4b52e0 Mon Sep 17 00:00:00 2001 From: Alison Shao <54658187+alisonshao@users.noreply.github.com> Date: Mon, 6 Apr 2026 15:32:09 -0700 Subject: [PATCH] [CI] Relax transformers MMLU threshold from 0.65 to 0.64 (#22210) Co-authored-by: Alison Shao --- test/registered/models/test_transformers_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/registered/models/test_transformers_models.py b/test/registered/models/test_transformers_models.py index 42416ad54..cc9a175b5 100644 --- a/test/registered/models/test_transformers_models.py +++ b/test/registered/models/test_transformers_models.py @@ -36,7 +36,7 @@ class TestTransformersFallbackEndpoint(CustomTestCase): timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, other_args=["--model-impl", "transformers"], ) - cls.mmlu_lower_bound = 0.65 + cls.mmlu_lower_bound = 0.64 cls.gsm8k_lower_bound = 0.65 @classmethod @@ -86,7 +86,7 @@ class TestTransformersFallbackTorchAO(TestTransformersFallbackEndpoint): "int4wo-128", ], ) - cls.mmlu_lower_bound = 0.65 + cls.mmlu_lower_bound = 0.64 cls.gsm8k_lower_bound = 0.65