[Bench] extend MMMU answer extractor with explicit-commit patterns (#24084)

benchmark/mmmu/eval_utils.py

@@ -273,22 +273,30 @@ def get_sampling_params(eval_args):
 
 
 # ----------- Process Multi-choice -------------
+# Patterns that explicitly commit to a single letter as the final answer.
+# Each captures the letter in group(1).  Matching uses ``re.IGNORECASE`` and
+# all matches are collected across patterns; the one with the latest offset
+# wins.
+_EXPLICIT_ANSWER_PATTERNS = (
+    # "answer: X" / "Final answer: X" (with optional bold/parens)
+    r"\banswer\s*:\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])",
+    # bare "X" / "(X)" on its own line at the end of the response
+    r"(?:^|\n)\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\.?\s*$",
+    # "\boxed{X}" (LaTeX boxed answer, common in math/CoT outputs)
+    r"\\boxed\{\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\}",
+    # "(the) answer is X" / "(the) correct answer is X"
+    r"\b(?:the\s+)?answer\s+is\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])",
+)
+
+
 def _parse_explicit_multi_choice_answer(response, all_choices):
     choice_map = {choice.upper(): choice for choice in all_choices}
     matches = []
-
-    answer_pattern = r"\banswer\s*:\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])"
-    for match in re.finditer(answer_pattern, response, flags=re.IGNORECASE):
+    for pattern in _EXPLICIT_ANSWER_PATTERNS:
+        for match in re.finditer(pattern, response, flags=re.IGNORECASE):
             candidate = match.group(1).upper()
             if candidate in choice_map:
                 matches.append((match.start(1), choice_map[candidate]))
-
-    final_letter_pattern = r"(?:^|\n)\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\.?\s*$"
-    for match in re.finditer(final_letter_pattern, response, flags=re.IGNORECASE):
-        candidate = match.group(1).upper()
-        if candidate in choice_map:
-            matches.append((match.start(1), choice_map[candidate]))
-
     return max(matches)[1] if matches else None
 
 

test/registered/unit/bench/test_mmmu_eval_utils.py

@@ -165,6 +165,67 @@ class TestMMMUEvalUtils(CustomTestCase):
 
         self.assertEqual(pred_ans, "B")
 
+    def test_parse_multi_choice_extracts_boxed_answer(self):
+        response = "After computing the integral the result lines up with \\boxed{C}."
+
+        pred_ans = self.eval_utils.parse_multi_choice_response(
+            response, ["A", "B", "C", "D"], self._index_to_answer()
+        )
+
+        self.assertEqual(pred_ans, "C")
+
+    def test_parse_multi_choice_extracts_the_answer_is(self):
+        response = "Reasoning about the diagram, the answer is D."
+
+        pred_ans = self.eval_utils.parse_multi_choice_response(
+            response, ["A", "B", "C", "D"], self._index_to_answer()
+        )
+
+        self.assertEqual(pred_ans, "D")
+
+    def test_parse_multi_choice_extracts_final_answer(self):
+        response = "Working through the steps...\nFinal answer: A"
+
+        pred_ans = self.eval_utils.parse_multi_choice_response(
+            response, ["A", "B", "C", "D"], self._index_to_answer()
+        )
+
+        self.assertEqual(pred_ans, "A")
+
+    def test_parse_multi_choice_extracts_correct_answer_phrase(self):
+        response = (
+            "(A) is wrong because the proportions do not match.\n"
+            "The correct answer is B."
+        )
+
+        pred_ans = self.eval_utils.parse_multi_choice_response(
+            response, ["A", "B", "C", "D"], self._index_to_answer()
+        )
+
+        self.assertEqual(pred_ans, "B")
+
+    def test_parse_multi_choice_ignores_parenthetical_option_mentions(self):
+        # Thinking-style outputs discuss/reject several options inside
+        # ``<think>...</think>`` using parenthetical mentions like ``(A)``,
+        # ``(B)``.  Those mentions must NOT match any explicit-commit
+        # pattern, otherwise the latest-match heuristic would pick up a
+        # rejected option from the thinking text.  Only the explicit
+        # ``Answer: D`` after ``</think>`` should win.
+        response = (
+            "<think>\n"
+            "Option (A) seems plausible but the diagram shows otherwise.\n"
+            "Maybe (B) given the labels — actually no, (B) is contradicted "
+            "by the second figure. Let me reconsider; the data points to D.\n"
+            "</think>\n"
+            "Answer: D"
+        )
+
+        pred_ans = self.eval_utils.parse_multi_choice_response(
+            response, ["A", "B", "C", "D"], self._index_to_answer()
+        )
+
+        self.assertEqual(pred_ans, "D")
+
     def _multiple_choice_sample(self, response):
         return {
             "id": "sample-1",