docs(minimax-m3): add MMMU-Pro accuracy to B200 benchmark card (#28668)

This commit is contained in:
Xinyuan Tong
2026-06-18 11:40:56 -07:00
committed by GitHub
parent f7632ef860
commit 61a8b42c00
2 changed files with 9 additions and 1 deletions
@@ -41,7 +41,7 @@ export const benchmarks = [
{ workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64, num_prompts: 128 }, { workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64, num_prompts: 128 },
ttft_ms: 1580, tpot_ms: 24.1, tokens_per_sec_per_gpu: 265 }, ttft_ms: 1580, tpot_ms: 24.1, tokens_per_sec_per_gpu: 265 },
], ],
accuracy: { gpqa_pct: 89.1, gsm8k_pct: 96.5 }, // 2026-06-15, sgl-eval --thinking, recommended sampling (temp 1.0/top_p 0.95), tp8. GSM8K full 1319 = 96.51% (greedy 96.89%). GPQA Diamond 198, n-repeats 4 = pass@1[avg-of-4] 89.14% +/-1.73% (pass@4 95.45%, majority@4 93.52%). accuracy: { gpqa_pct: 89.1, gsm8k_pct: 96.5, mmmu_pro_pct: 72.7 }, // 2026-06-15, sgl-eval --thinking, recommended sampling (temp 1.0/top_p 0.95), tp8. GSM8K full 1319 = 96.51% (greedy 96.89%). GPQA Diamond 198, n-repeats 4 = pass@1[avg-of-4] 89.14% +/-1.73% (pass@4 95.45%, majority@4 93.52%). MMMU-Pro 2026-06-18, sgl-eval "standard (10 options)" test split, full 1730, single-shot 72.66% (thinking, temp 1.0/top_p 0.95).
}, },
{ {
// Hopper H200: bf16 build (MXFP8 is Blackwell-only) at tp8, built-in Triton // Hopper H200: bf16 build (MXFP8 is Blackwell-only) at tp8, built-in Triton
@@ -71,6 +71,13 @@ sgl-eval run gpqa \\
--model {{MODEL_NAME}} \\ --model {{MODEL_NAME}} \\
--temperature 1.0 --top-p 0.95 \\ --temperature 1.0 --top-p 0.95 \\
--thinking --n-repeats 4 --max-tokens 40960`, --thinking --n-repeats 4 --max-tokens 40960`,
mmmu_pro_pct:
`pip install git+https://github.com/sgl-project/sgl-eval
sgl-eval run mmmu_pro \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--model {{MODEL_NAME}} \\
--temperature 1.0 --top-p 0.95 \\
--thinking`,
}, },
numPromptsByConc: { 24: 24, 64: 128 }, numPromptsByConc: { 24: 24, 64: 128 },
}, },
@@ -78,6 +85,7 @@ sgl-eval run gpqa \\
accuracyLabels: [ accuracyLabels: [
["gpqa_pct", "GPQA Diamond", "%"], ["gpqa_pct", "GPQA Diamond", "%"],
["gsm8k_pct", "GSM8K", "%"], ["gsm8k_pct", "GSM8K", "%"],
["mmmu_pro_pct", "MMMU-Pro", "%"],
], ],
dockerImages: { dockerImages: {