docs: cookbook benchmark accuracy labels come from the model config (no engine default) (#27842)
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
66989a7642
commit
43835b5ba5
@@ -20,6 +20,9 @@
|
||||
// benchmarkCommands optional — powers the "⚡ Reproduce" modal (speed +
|
||||
// per-eval accuracy templates)
|
||||
// defaultAccuracy optional — per-variant accuracy merged under cell.accuracy
|
||||
// accuracyLabels [key, label, unit][] — the eval set shown in the
|
||||
// benchmark card + "⚡ Reproduce". NO engine default:
|
||||
// required whenever benchmarks carry accuracy data
|
||||
// multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines
|
||||
// dockerImages optional — per-hw image for `docker run` mode
|
||||
// github optional — "Submit verified cell" issue-template overrides
|
||||
@@ -530,13 +533,12 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
return cmd;
|
||||
};
|
||||
|
||||
// Accuracy labels: [field-key, display-label, unit]. Keys must match the
|
||||
// `accuracy` fields in the benchmarks file + `benchmarkCommands.accuracy`.
|
||||
const ACCURACY_LABELS = [
|
||||
["gpqa_pct", "GPQA Diamond", "%"],
|
||||
["aime25_pct", "AIME25", "%"],
|
||||
["gsm8k_pct", "GSM8K (1-shot)", "%"],
|
||||
];
|
||||
// Accuracy labels: [field-key, display-label, unit]. Declared per model via
|
||||
// `config.accuracyLabels` — the engine ships NO default eval set. A config
|
||||
// without it renders no accuracy rows (and no Accuracy section in the
|
||||
// "⚡ Reproduce" modal). Keys must match the `accuracy` fields in the
|
||||
// benchmarks file + `benchmarkCommands.accuracy`.
|
||||
const ACCURACY_LABELS = config.accuracyLabels || [];
|
||||
|
||||
const renderBenchmarkCard = (entry) => {
|
||||
// [key, label, unit, compute?]. Optional compute(measurement) supplies
|
||||
|
||||
@@ -122,6 +122,14 @@ sgl-eval run aime25 \\
|
||||
pro: { gpqa_pct: 90.1, aime25_pct: 97.5, gsm8k_pct: 96.13 },
|
||||
},
|
||||
|
||||
// The eval set rendered in the benchmark card + "⚡ Reproduce" (the engine
|
||||
// ships no default — every config declares its own).
|
||||
accuracyLabels: [
|
||||
["gpqa_pct", "GPQA Diamond", "%"],
|
||||
["aime25_pct", "AIME25", "%"],
|
||||
["gsm8k_pct", "GSM8K (1-shot)", "%"],
|
||||
],
|
||||
|
||||
// Prepended as `# ...` comments above multi-node commands.
|
||||
multiNodeHints: {
|
||||
gb200: [
|
||||
|
||||
Reference in New Issue
Block a user