docs: cookbook benchmark accuracy labels come from the model config (no engine default) (#27842)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
zijiexia
2026-06-10 18:27:38 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent 66989a7642
commit 43835b5ba5
5 changed files with 33 additions and 10 deletions
+9 -7
View File
@@ -20,6 +20,9 @@
// benchmarkCommands optional — powers the "⚡ Reproduce" modal (speed +
// per-eval accuracy templates)
// defaultAccuracy optional — per-variant accuracy merged under cell.accuracy
// accuracyLabels [key, label, unit][] — the eval set shown in the
// benchmark card + "⚡ Reproduce". NO engine default:
// required whenever benchmarks carry accuracy data
// multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines
// dockerImages optional — per-hw image for `docker run` mode
// github optional — "Submit verified cell" issue-template overrides
@@ -530,13 +533,12 @@ export const Deployment = ({ config, benchmarks }) => {
return cmd;
};
// Accuracy labels: [field-key, display-label, unit]. Keys must match the
// `accuracy` fields in the benchmarks file + `benchmarkCommands.accuracy`.
const ACCURACY_LABELS = [
["gpqa_pct", "GPQA Diamond", "%"],
["aime25_pct", "AIME25", "%"],
["gsm8k_pct", "GSM8K (1-shot)", "%"],
];
// Accuracy labels: [field-key, display-label, unit]. Declared per model via
// `config.accuracyLabels` — the engine ships NO default eval set. A config
// without it renders no accuracy rows (and no Accuracy section in the
// "⚡ Reproduce" modal). Keys must match the `accuracy` fields in the
// benchmarks file + `benchmarkCommands.accuracy`.
const ACCURACY_LABELS = config.accuracyLabels || [];
const renderBenchmarkCard = (entry) => {
// [key, label, unit, compute?]. Optional compute(measurement) supplies
@@ -122,6 +122,14 @@ sgl-eval run aime25 \\
pro: { gpqa_pct: 90.1, aime25_pct: 97.5, gsm8k_pct: 96.13 },
},
// The eval set rendered in the benchmark card + "⚡ Reproduce" (the engine
// ships no default — every config declares its own).
accuracyLabels: [
["gpqa_pct", "GPQA Diamond", "%"],
["aime25_pct", "AIME25", "%"],
["gsm8k_pct", "GSM8K (1-shot)", "%"],
],
// Prepended as `# ...` comments above multi-node commands.
multiNodeHints: {
gb200: [