diff --git a/benchmark/mmmu/eval_utils.py b/benchmark/mmmu/eval_utils.py
index d0ea42363..33a592551 100644
--- a/benchmark/mmmu/eval_utils.py
+++ b/benchmark/mmmu/eval_utils.py
@@ -273,22 +273,30 @@ def get_sampling_params(eval_args):
# ----------- Process Multi-choice -------------
+# Patterns that explicitly commit to a single letter as the final answer.
+# Each captures the letter in group(1). Matching uses ``re.IGNORECASE`` and
+# all matches are collected across patterns; the one with the latest offset
+# wins.
+_EXPLICIT_ANSWER_PATTERNS = (
+ # "answer: X" / "Final answer: X" (with optional bold/parens)
+ r"\banswer\s*:\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])",
+ # bare "X" / "(X)" on its own line at the end of the response
+ r"(?:^|\n)\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\.?\s*$",
+ # "\boxed{X}" (LaTeX boxed answer, common in math/CoT outputs)
+ r"\\boxed\{\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\}",
+ # "(the) answer is X" / "(the) correct answer is X"
+ r"\b(?:the\s+)?answer\s+is\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])",
+)
+
+
def _parse_explicit_multi_choice_answer(response, all_choices):
choice_map = {choice.upper(): choice for choice in all_choices}
matches = []
-
- answer_pattern = r"\banswer\s*:\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])"
- for match in re.finditer(answer_pattern, response, flags=re.IGNORECASE):
- candidate = match.group(1).upper()
- if candidate in choice_map:
- matches.append((match.start(1), choice_map[candidate]))
-
- final_letter_pattern = r"(?:^|\n)\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\.?\s*$"
- for match in re.finditer(final_letter_pattern, response, flags=re.IGNORECASE):
- candidate = match.group(1).upper()
- if candidate in choice_map:
- matches.append((match.start(1), choice_map[candidate]))
-
+ for pattern in _EXPLICIT_ANSWER_PATTERNS:
+ for match in re.finditer(pattern, response, flags=re.IGNORECASE):
+ candidate = match.group(1).upper()
+ if candidate in choice_map:
+ matches.append((match.start(1), choice_map[candidate]))
return max(matches)[1] if matches else None
diff --git a/test/registered/unit/bench/test_mmmu_eval_utils.py b/test/registered/unit/bench/test_mmmu_eval_utils.py
index 36f2bb695..fe6a16ae3 100644
--- a/test/registered/unit/bench/test_mmmu_eval_utils.py
+++ b/test/registered/unit/bench/test_mmmu_eval_utils.py
@@ -165,6 +165,67 @@ class TestMMMUEvalUtils(CustomTestCase):
self.assertEqual(pred_ans, "B")
+ def test_parse_multi_choice_extracts_boxed_answer(self):
+ response = "After computing the integral the result lines up with \\boxed{C}."
+
+ pred_ans = self.eval_utils.parse_multi_choice_response(
+ response, ["A", "B", "C", "D"], self._index_to_answer()
+ )
+
+ self.assertEqual(pred_ans, "C")
+
+ def test_parse_multi_choice_extracts_the_answer_is(self):
+ response = "Reasoning about the diagram, the answer is D."
+
+ pred_ans = self.eval_utils.parse_multi_choice_response(
+ response, ["A", "B", "C", "D"], self._index_to_answer()
+ )
+
+ self.assertEqual(pred_ans, "D")
+
+ def test_parse_multi_choice_extracts_final_answer(self):
+ response = "Working through the steps...\nFinal answer: A"
+
+ pred_ans = self.eval_utils.parse_multi_choice_response(
+ response, ["A", "B", "C", "D"], self._index_to_answer()
+ )
+
+ self.assertEqual(pred_ans, "A")
+
+ def test_parse_multi_choice_extracts_correct_answer_phrase(self):
+ response = (
+ "(A) is wrong because the proportions do not match.\n"
+ "The correct answer is B."
+ )
+
+ pred_ans = self.eval_utils.parse_multi_choice_response(
+ response, ["A", "B", "C", "D"], self._index_to_answer()
+ )
+
+ self.assertEqual(pred_ans, "B")
+
+ def test_parse_multi_choice_ignores_parenthetical_option_mentions(self):
+ # Thinking-style outputs discuss/reject several options inside
+ # ``...`` using parenthetical mentions like ``(A)``,
+ # ``(B)``. Those mentions must NOT match any explicit-commit
+ # pattern, otherwise the latest-match heuristic would pick up a
+ # rejected option from the thinking text. Only the explicit
+ # ``Answer: D`` after ```` should win.
+ response = (
+ "\n"
+ "Option (A) seems plausible but the diagram shows otherwise.\n"
+ "Maybe (B) given the labels — actually no, (B) is contradicted "
+ "by the second figure. Let me reconsider; the data points to D.\n"
+ "\n"
+ "Answer: D"
+ )
+
+ pred_ans = self.eval_utils.parse_multi_choice_response(
+ response, ["A", "B", "C", "D"], self._index_to_answer()
+ )
+
+ self.assertEqual(pred_ans, "D")
+
def _multiple_choice_sample(self, response):
return {
"id": "sample-1",