docs(minimax-m3): add MMMU-Pro accuracy to B200 benchmark card (#28668)
This commit is contained in:
@@ -41,7 +41,7 @@ export const benchmarks = [
|
||||
{ workload: { dataset: "random", isl: 2048, osl: 256, max_concurrency: 64, num_prompts: 128 },
|
||||
ttft_ms: 1580, tpot_ms: 24.1, tokens_per_sec_per_gpu: 265 },
|
||||
],
|
||||
accuracy: { gpqa_pct: 89.1, gsm8k_pct: 96.5 }, // 2026-06-15, sgl-eval --thinking, recommended sampling (temp 1.0/top_p 0.95), tp8. GSM8K full 1319 = 96.51% (greedy 96.89%). GPQA Diamond 198, n-repeats 4 = pass@1[avg-of-4] 89.14% +/-1.73% (pass@4 95.45%, majority@4 93.52%).
|
||||
accuracy: { gpqa_pct: 89.1, gsm8k_pct: 96.5, mmmu_pro_pct: 72.7 }, // 2026-06-15, sgl-eval --thinking, recommended sampling (temp 1.0/top_p 0.95), tp8. GSM8K full 1319 = 96.51% (greedy 96.89%). GPQA Diamond 198, n-repeats 4 = pass@1[avg-of-4] 89.14% +/-1.73% (pass@4 95.45%, majority@4 93.52%). MMMU-Pro 2026-06-18, sgl-eval "standard (10 options)" test split, full 1730, single-shot 72.66% (thinking, temp 1.0/top_p 0.95).
|
||||
},
|
||||
{
|
||||
// Hopper H200: bf16 build (MXFP8 is Blackwell-only) at tp8, built-in Triton
|
||||
|
||||
@@ -71,6 +71,13 @@ sgl-eval run gpqa \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--temperature 1.0 --top-p 0.95 \\
|
||||
--thinking --n-repeats 4 --max-tokens 40960`,
|
||||
mmmu_pro_pct:
|
||||
`pip install git+https://github.com/sgl-project/sgl-eval
|
||||
sgl-eval run mmmu_pro \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--temperature 1.0 --top-p 0.95 \\
|
||||
--thinking`,
|
||||
},
|
||||
numPromptsByConc: { 24: 24, 64: 128 },
|
||||
},
|
||||
@@ -78,6 +85,7 @@ sgl-eval run gpqa \\
|
||||
accuracyLabels: [
|
||||
["gpqa_pct", "GPQA Diamond", "%"],
|
||||
["gsm8k_pct", "GSM8K", "%"],
|
||||
["mmmu_pro_pct", "MMMU-Pro", "%"],
|
||||
],
|
||||
|
||||
dockerImages: {
|
||||
|
||||
Reference in New Issue
Block a user