diff --git a/.claude/skills/cookbook-add-model/references/authoring-reference.md b/.claude/skills/cookbook-add-model/references/authoring-reference.md index 56d2177e2..770a063c0 100644 --- a/.claude/skills/cookbook-add-model/references/authoring-reference.md +++ b/.claude/skills/cookbook-add-model/references/authoring-reference.md @@ -47,7 +47,8 @@ the full contract): | `dockerImages` | `{[hwId]: string}` | Per-hw image name for `docker run` framing. **Ask the user which sglang build the recipes ran on; don't guess a supporting release.** Falls back to `lmsysorg/sglang:dev` if missing — also the sensible default when unsure. | | `playgroundFeatures` | `{[axisId]: {...}}` | Opts into the Playground widget. See §2.3. | | `benchmarkCommands` | `{speed: string, accuracy: {[accKey]: string \| {[variant]: string}}, numPromptsByConc?: {[c]: number}}` | Powers the benchmark card's **"⚡ Reproduce"** modal. `speed` is ONE `bench_serving` template; the engine fills `{{DATASET}}`/`{{ISL}}`/`{{OSL}}` from each cell's `speed[].workload`, the chip-picked `{{MAX_CONCURRENCY}}`, and `{{NUM_PROMPTS}}` (resolved `workload.num_prompts ?? numPromptsByConc[c] ?? max(c*2, 200)`). `accuracy` maps an accuracy field (e.g. `gsm8k_pct`) to a per-eval template — a string, OR a `{flash, pro, …}` object keyed by variant when the command differs per variant (e.g. GPQA/AIME `--max-tokens`). The modal renders a chip per eval (one command area, like Speed). Both also use `{{MODEL_NAME}}` + `{{CURL_HOST}}`/`{{CURL_PORT}}` like `curl`. Optional; the button only appears when this AND `benchmarks` are present. | -| `defaultAccuracy` | `{[variant]: {[accKey]: number}}` | Model-level accuracy applied to **every** cell of a variant (e.g. GPQA Diamond / AIME25 — hardware-independent). Merged UNDER each cell's measured `accuracy` (a per-cell value wins), so you set a variant's score once instead of copying it onto every benchmark entry. Keys must match `ACCURACY_LABELS` + `benchmarkCommands.accuracy`. | +| `defaultAccuracy` | `{[variant]: {[accKey]: number}}` | Model-level accuracy applied to **every** cell of a variant (e.g. GPQA Diamond / AIME25 — hardware-independent). Merged UNDER each cell's measured `accuracy` (a per-cell value wins), so you set a variant's score once instead of copying it onto every benchmark entry. Keys must match `accuracyLabels` (below) + `benchmarkCommands.accuracy`. | +| `accuracyLabels` | `[key, label, unit][]` | The eval set rendered in the benchmark card and the "⚡ Reproduce" modal — **the engine ships no default**, every config declares its own (e.g. DSv4: GPQA/AIME25/GSM8K; Qwen3.5: GSM8K/MMMU). Required whenever the benchmarks carry accuracy data; without it the accuracy rows silently don't render. Every key used in `benchmarks[].accuracy`, `defaultAccuracy`, and `benchmarkCommands.accuracy` must appear here. | | `github` | `{owner?, repo?, issueTemplate?, cookbookModel?}` | Overrides for the "Submit verified cell" CTA in the playground. Defaults: `sgl-project/sglang` + `3-playground-verified-cell.yml` + `"deepseek-ai/deepseek-v4"`. Set `cookbookModel` to the model's HF id (`/`); it prefills the issue template's free-form `model` input when the issue opens. **Don't prune this block** — without it the engine falls back to `deepseek-ai/deepseek-v4` and submissions from your page get mislabeled. | ## 2.2 Author the 5-dim matrix (`cells[]`) diff --git a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl index 0946237e9..9f6e49f1a 100644 --- a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -108,11 +108,20 @@ sgl-eval run gsm8k \\ // OPTIONAL — per-variant accuracy applied to EVERY cell of a variant (hardware- // independent, e.g. GPQA/AIME). Per-cell `accuracy` overrides. Keys must match - // ACCURACY_LABELS + benchmarkCommands.accuracy. Delete if no numbers yet. + // the effective accuracy labels + benchmarkCommands.accuracy. Delete if no numbers yet. defaultAccuracy: { default: { gsm8k_pct: null }, }, + // The eval set rendered in the benchmark card + "⚡ Reproduce" — the engine + // ships NO default; required whenever the benchmarks carry accuracy data + // (without it the accuracy rows silently don't render). [key, label, unit] + // tuples; keys must match benchmarks[].accuracy + defaultAccuracy + + // benchmarkCommands.accuracy. Delete only if there are no accuracy numbers. + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ], + // OPTIONAL — `# ...` hint lines prepended to multi-node commands, ONLY for hw // whose fabric needs manual NIC env (e.g. gb200 NVL72/MNNVL). NOT every multi-N // hw needs this — standard-IB DeepEP / Marlin multi-node don't. Delete if unused. diff --git a/.claude/skills/cookbook-review-pr/SKILL.md b/.claude/skills/cookbook-review-pr/SKILL.md index 1948db070..6a5bfe72b 100644 --- a/.claude/skills/cookbook-review-pr/SKILL.md +++ b/.claude/skills/cookbook-review-pr/SKILL.md @@ -78,7 +78,10 @@ than restating. ### 4. Benchmarks - Each `benchmarks[]` entry's `match` tuple corresponds to a real cell. -- `defaultAccuracy` keys ∈ `ACCURACY_LABELS` (and `benchmarkCommands.accuracy`). +- `accuracyLabels` is present whenever the benchmarks carry accuracy data — the engine + ships NO default eval set; without it the accuracy rows silently don't render. + `defaultAccuracy` / per-cell `accuracy` / `benchmarkCommands.accuracy` keys all + ∈ `config.accuracyLabels`. - A benchmark's quantization must match a variant actually listed — `(BF16)` on a model that only released FP8/FP4 is a factual bug. - `benchmarkCommands.speed` is `python3 -m sglang.bench_serving` (the workload), separate diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index 51b820ee6..fe79f1ca3 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -20,6 +20,9 @@ // benchmarkCommands optional — powers the "⚡ Reproduce" modal (speed + // per-eval accuracy templates) // defaultAccuracy optional — per-variant accuracy merged under cell.accuracy +// accuracyLabels [key, label, unit][] — the eval set shown in the +// benchmark card + "⚡ Reproduce". NO engine default: +// required whenever benchmarks carry accuracy data // multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines // dockerImages optional — per-hw image for `docker run` mode // github optional — "Submit verified cell" issue-template overrides @@ -530,13 +533,12 @@ export const Deployment = ({ config, benchmarks }) => { return cmd; }; - // Accuracy labels: [field-key, display-label, unit]. Keys must match the - // `accuracy` fields in the benchmarks file + `benchmarkCommands.accuracy`. - const ACCURACY_LABELS = [ - ["gpqa_pct", "GPQA Diamond", "%"], - ["aime25_pct", "AIME25", "%"], - ["gsm8k_pct", "GSM8K (1-shot)", "%"], - ]; + // Accuracy labels: [field-key, display-label, unit]. Declared per model via + // `config.accuracyLabels` — the engine ships NO default eval set. A config + // without it renders no accuracy rows (and no Accuracy section in the + // "⚡ Reproduce" modal). Keys must match the `accuracy` fields in the + // benchmarks file + `benchmarkCommands.accuracy`. + const ACCURACY_LABELS = config.accuracyLabels || []; const renderBenchmarkCard = (entry) => { // [key, label, unit, compute?]. Optional compute(measurement) supplies diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index 05b59b341..1118d57dd 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -122,6 +122,14 @@ sgl-eval run aime25 \\ pro: { gpqa_pct: 90.1, aime25_pct: 97.5, gsm8k_pct: 96.13 }, }, + // The eval set rendered in the benchmark card + "⚡ Reproduce" (the engine + // ships no default — every config declares its own). + accuracyLabels: [ + ["gpqa_pct", "GPQA Diamond", "%"], + ["aime25_pct", "AIME25", "%"], + ["gsm8k_pct", "GSM8K (1-shot)", "%"], + ], + // Prepended as `# ...` comments above multi-node commands. multiNodeHints: { gb200: [