minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Bench] extend MMMU answer extractor with explicit-commit patterns (#24084)

Browse Source
This commit is contained in:
Yihao Wang
2026-04-30 19:08:08 -07:00
committed by GitHub
parent 1742bfb610
commit 0acc569edd
2 changed files with 82 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmark/mmmu/eval_utils.py
+21 -13
View File
@@ -273,22 +273,30 @@ def get_sampling_params(eval_args):
# ----------- Process Multi-choice -------------
# Patterns that explicitly commit to a single letter as the final answer.
# Each captures the letter in group(1). Matching uses ``re.IGNORECASE`` and
# all matches are collected across patterns; the one with the latest offset
# wins.
_EXPLICIT_ANSWER_PATTERNS = (
# "answer: X" / "Final answer: X" (with optional bold/parens)
r"\banswer\s*:\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])",
# bare "X" / "(X)" on its own line at the end of the response
r"(?:^|\n)\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\.?\s*$",
# "\boxed{X}" (LaTeX boxed answer, common in math/CoT outputs)
r"\\boxed\{\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\}",
# "(the) answer is X" / "(the) correct answer is X"
r"\b(?:the\s+)?answer\s+is\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])",
)
def _parse_explicit_multi_choice_answer(response, all_choices):
choice_map = {choice.upper(): choice for choice in all_choices}
matches = []
answer_pattern = r"\banswer\s*:\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}(?![A-Za-z])"
for match in re.finditer(answer_pattern, response, flags=re.IGNORECASE):
candidate = match.group(1).upper()
if candidate in choice_map:
matches.append((match.start(1), choice_map[candidate]))
final_letter_pattern = r"(?:^|\n)\s*\*{0,2}\s*\(?([A-Z])\)?\s*\*{0,2}\s*\.?\s*$"
for match in re.finditer(final_letter_pattern, response, flags=re.IGNORECASE):
candidate = match.group(1).upper()
if candidate in choice_map:
matches.append((match.start(1), choice_map[candidate]))
for pattern in _EXPLICIT_ANSWER_PATTERNS:
for match in re.finditer(pattern, response, flags=re.IGNORECASE):
candidate = match.group(1).upper()
if candidate in choice_map:
matches.append((match.start(1), choice_map[candidate]))
return max(matches)[1] if matches else None