diff --git a/.claude/skills/cookbook-add-model/references/authoring-reference.md b/.claude/skills/cookbook-add-model/references/authoring-reference.md index d2eedd6f8..3e70d54bc 100644 --- a/.claude/skills/cookbook-add-model/references/authoring-reference.md +++ b/.claude/skills/cookbook-add-model/references/authoring-reference.md @@ -46,10 +46,10 @@ the full contract): | `multiNodeHints` | `{[hwId]: string[]}` | Lines prepended as `# ...` comments to multi-node commands (env-var hints). Per-hw, and only for hw whose **cluster fabric needs manual NIC config** (e.g. `gb200` NVL72/MNNVL → NVSHMEM/Gloo hints). NOT every multi-N hw needs an entry — standard-IB DeepEP (h200) auto-detects the HCA, and Marlin multi-node (h100) uses no DeepEP/NVSHMEM at all. | | `dockerImages` | `{[key]: string}` | Image for `docker run` framing, keyed by `hw\|quant` (most specific) then `hw`. Use a `hw\|quant` key only when one quant on a shared GPU needs a different image (e.g. an NVFP4 dev build on b300/gb300 while FP8/BF16 stay on the release image); otherwise key by plain `hw`. **Ask the user which sglang build the recipes ran on; don't guess a supporting release.** Falls back to `lmsysorg/sglang:dev` if missing — also the sensible default when unsure. | | `playgroundFeatures` | `{[axisId]: {...}}` | Opts into the Playground widget. See §2.3. | -| `benchmarkCommands` | `{speed: string, accuracy: {[accKey]: string \| {[variant]: string}}, numPromptsByConc?: {[c]: number}}` | Powers the benchmark card's **"⚡ Reproduce"** modal. `speed` is ONE `bench_serving` template; the engine fills `{{DATASET}}`/`{{ISL}}`/`{{OSL}}` from each cell's `speed[].workload`, the chip-picked `{{MAX_CONCURRENCY}}`, and `{{NUM_PROMPTS}}` (resolved `workload.num_prompts ?? numPromptsByConc[c] ?? max(c*2, 200)`). `accuracy` maps an accuracy field (e.g. `gsm8k_pct`) to a per-eval template — a string, OR a `{flash, pro, …}` object keyed by variant when the command differs per variant (e.g. GPQA/AIME `--max-tokens`). The modal renders a chip per eval (one command area, like Speed). Both also use `{{MODEL_NAME}}` + `{{CURL_HOST}}`/`{{CURL_PORT}}` like `curl`. Optional; the button only appears when this AND `benchmarks` are present. | +| `benchmarkCommands` | `{speed: string, accuracy: {[accKey]: string \| {[variant]: string}}, numPromptsByConc?: {[c]: number}}` | Powers the benchmark card's **"⚡ Reproduce"** modal. `speed` is ONE `bench_serving` template; the engine fills `{{DATASET}}`/`{{ISL}}`/`{{OSL}}` from each cell's `speed[].workload`, the chip-picked `{{MAX_CONCURRENCY}}`, and `{{NUM_PROMPTS}}` (resolved `workload.num_prompts ?? numPromptsByConc[c] ?? max(c*2, 200)`). `accuracy` maps an accuracy field (e.g. `gsm8k_pct`) to a per-eval template — a string, OR a `{flash, pro, …}` object keyed by variant when the command differs per variant (e.g. GPQA/AIME `--max-tokens`). The modal renders a chip per eval (one command area, like Speed). Both also use `{{MODEL_NAME}}` + `{{CURL_HOST}}`/`{{CURL_PORT}}` like `curl`. `speed` should carry `--flush-cache` (bench_serving's `random` prompts are deterministic — warm reruns hit the radix cache and inflate throughput; measure cache-cold). Optional; the button only appears when this AND `benchmarks` are present. | | `defaultAccuracy` | `{[variant]: {[accKey]: number}}` | Model-level accuracy applied to **every** cell of a variant (e.g. GPQA Diamond / AIME25 — hardware-independent). Merged UNDER each cell's measured `accuracy` (a per-cell value wins), so you set a variant's score once instead of copying it onto every benchmark entry. Keys must match `accuracyLabels` (below) + `benchmarkCommands.accuracy`. | | `accuracyLabels` | `[key, label, unit][]` | The eval set rendered in the benchmark card and the "⚡ Reproduce" modal — **the engine ships no default**, every config declares its own (e.g. DSv4: GPQA/AIME25/GSM8K; Qwen3.5: GSM8K/MMMU). Required whenever the benchmarks carry accuracy data; without it the accuracy rows silently don't render. Every key used in `benchmarks[].accuracy`, `defaultAccuracy`, and `benchmarkCommands.accuracy` must appear here. | -| `latencyPercentile` | `"Mean" \| "P50"` | Optional, **temporary**; the percentile the benchmark TTFT/TPOT values are. **Default `"P50"`** — the card renders `TTFT ()` / `TPOT ()`. Set `"Mean"` only for legacy data recorded as Mean (being re-measured to P50). `tokens_per_sec_per_gpu` is stored as **total (in+out)/GPU** = `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. | +| `latencyPercentile` | `"Mean" \| "P50"` | Optional, **temporary**; the percentile the benchmark TTFT/TPOT values are. **Default `"P50"`** — the card renders `TTFT ()` / `TPOT ()`. Set `"Mean"` only for legacy data recorded as Mean (being re-measured to P50). A benchmarks entry may carry its own `latencyPercentile` to override the page value per cell (entry → config → `"P50"`). `tokens_per_sec_per_gpu` is stored as **total (in+out)/GPU** = `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. | | `github` | `{owner?, repo?, issueTemplate?, cookbookModel?}` | Overrides for the "Submit verified cell" CTA in the playground. Defaults: `sgl-project/sglang` + `3-playground-verified-cell.yml` + `"deepseek-ai/deepseek-v4"`. Set `cookbookModel` to the model's HF id (`/`); it prefills the issue template's free-form `model` input when the issue opens. **Don't prune this block** — without it the engine falls back to `deepseek-ai/deepseek-v4` and submissions from your page get mislabeled. | ## 2.2 Author the 5-dim matrix (`cells[]`) diff --git a/.claude/skills/cookbook-add-model/templates/benchmarks.jsx.tmpl b/.claude/skills/cookbook-add-model/templates/benchmarks.jsx.tmpl index 28193d23a..2aa82dcdd 100644 --- a/.claude/skills/cookbook-add-model/templates/benchmarks.jsx.tmpl +++ b/.claude/skills/cookbook-add-model/templates/benchmarks.jsx.tmpl @@ -7,7 +7,8 @@ // "pending" until an entry has a non-null speed metric or accuracy. Speed shape: // speed: [{ workload: {dataset, isl, osl, max_concurrency}, ttft_ms, tpot_ms, // tokens_per_sec_per_gpu }, ...] -// - ttft_ms/tpot_ms are P50 (median); set config.latencyPercentile ("P50" default, or "Mean"). +// - ttft_ms/tpot_ms are P50 (median); set config.latencyPercentile ("P50" default, or "Mean"); +// an entry-level latencyPercentile overrides the page value per cell. // - tokens_per_sec_per_gpu = total (in+out) tok/s/GPU // (= output tok/s ÷ GPUs × (isl+osl)/osl). interactivity is derived = 1000/TPOT (tokens/s/user). // Per-cell `accuracy: { : }` overrides the config's defaultAccuracy. diff --git a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl index e3d482213..21a2ba678 100644 --- a/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl +++ b/.claude/skills/cookbook-add-model/templates/config.jsx.tmpl @@ -97,7 +97,8 @@ export const config = { --model {{MODEL_NAME}} \\ --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} \\ - --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ + --flush-cache`, // One entry per accuracy field. A value is a string, OR a {[variant]: string} // object when the command differs per variant. Keys must match ACCURACY_LABELS // in _deployment.jsx + the per-cell/defaultAccuracy keys. diff --git a/.claude/skills/cookbook-migrate-model/SKILL.md b/.claude/skills/cookbook-migrate-model/SKILL.md index 9c4c4a1ae..7b0983ca0 100644 --- a/.claude/skills/cookbook-migrate-model/SKILL.md +++ b/.claude/skills/cookbook-migrate-model/SKILL.md @@ -177,7 +177,8 @@ One entry per measured block only (cells without entries already render total (in+out) tok/s/GPU = `output tok/s ÷ (tp × nnodes) × (isl+osl)/osl` — stored directly (the card shows it as-is). TTFT/TPOT take the P50 (median) rows; set `config.latencyPercentile` (default `"P50"`; use -`"Mean"` only for legacy Mean-recorded data — temporary, being migrated to P50). +`"Mean"` only for legacy Mean-recorded data — temporary, being migrated to P50; an +entry-level `latencyPercentile` overrides the page value per cell). Put the workload's `num_prompts` into `workload`. **`config.accuracyLabels` is required whenever the benchmarks carry accuracy data** — the engine ships no default eval set diff --git a/.claude/skills/cookbook-review-pr/SKILL.md b/.claude/skills/cookbook-review-pr/SKILL.md index 782038340..1bbce320d 100644 --- a/.claude/skills/cookbook-review-pr/SKILL.md +++ b/.claude/skills/cookbook-review-pr/SKILL.md @@ -115,12 +115,17 @@ than restating. - A benchmark's quantization must match a variant actually listed — `(BF16)` on a model that only released FP8/FP4 is a factual bug. - `benchmarkCommands.speed` is `python3 -m sglang.bench_serving` (the workload), separate - from the `sglang serve` deploy command. + from the `sglang serve` deploy command, and should carry `--flush-cache`: bench_serving's + `random` prompts are deterministic, so a warm rerun hits the radix cache and inflates + throughput — speed numbers are measured cache-cold. - `sglang_version` is a real build the author ran (a release, or `dev`/nightly) — not a guessed/placeholder value (no leftover `0.0.0`). - **Latency percentile**: `config.latencyPercentile` (default `"P50"`, or `"Mean"`) matches the - percentile the TTFT/TPOT values actually are — the card renders `TTFT ()`. (`"Mean"` is - temporary — legacy data is being re-measured to P50.) + percentile the TTFT/TPOT values actually are — the card renders `TTFT ()`. A benchmarks + entry may carry its own `latencyPercentile` to override the page value per cell + (entry → config → `"P50"`): on a P50 page, kept legacy Mean cells must set it — a + `sglang_version` tag alone doesn't convey the percentile. (`"Mean"` is temporary — legacy + data is being re-measured to P50.) - **Throughput convention**: `tokens_per_sec_per_gpu` is stored as **total (in+out)/GPU** = `output tok/s/GPU × (isl+osl)/osl`, shown by the card as-is. Flag output-only values. - **Consistent accuracy harness across entries**: every value under one `accuracyLabels` diff --git a/docs_new/src/snippets/_deployment.jsx b/docs_new/src/snippets/_deployment.jsx index 32ad7606c..8d86d6dc7 100644 --- a/docs_new/src/snippets/_deployment.jsx +++ b/docs_new/src/snippets/_deployment.jsx @@ -25,6 +25,8 @@ // required whenever benchmarks carry accuracy data // latencyPercentile optional, TEMPORARY — "Mean" | "P50" (default "P50"); the // percentile the TTFT/TPOT values are, shown as "TTFT ()". +// A benchmarks entry may carry its own latencyPercentile to +// override the page value per cell (entry → config → "P50"). // Legacy "Mean" data is being re-measured to P50; drop once done // multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines // dockerImages optional — `docker run` image, keyed by `hw|quant` @@ -577,7 +579,7 @@ export const Deployment = ({ config, benchmarks }) => { const renderBenchmarkCard = (entry) => { // [key, label, unit, compute?]. Optional compute(measurement) supplies // derived metrics (preferred over measurement[key] when present). - const pct = config.latencyPercentile || "P50"; + const pct = (entry && entry.latencyPercentile) || config.latencyPercentile || "P50"; const SPEED_LABELS = [ ["ttft_ms", `TTFT (${pct})`, "ms"], ["tpot_ms", `TPOT (${pct})`, "ms"], diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx index 7372cdd15..5f2e08bcd 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4-benchmarks.jsx @@ -1,113 +1,181 @@ // DeepSeek-V4 per-cell benchmark numbers, keyed by the same `match` tuple as // deepseek-v4.jsx cells. See _deployment.jsx for the speed/accuracy schema. -// Measured on sglang v0.5.12.post1. -// tokens_per_sec_per_gpu is total (input+output) tok/s/GPU: fp4/fp8 = measured -// output/GPU × (isl+osl)/osl; nvfp4 was measured as total already. +// Measured on sglang v0.5.15 / v0.5.15.post1 (per-cell sglang_version). +// tokens_per_sec_per_gpu is total (input+output) tok/s/GPU = output/GPU × (isl+osl)/osl. export const benchmarks = [ // ==================================================================== // B200 + FP4 // ==================================================================== { match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 87, tpot_ms: 3.68, tokens_per_sec_per_gpu: 585 }, + ttft_ms: 302, tpot_ms: 2.91, tokens_per_sec_per_gpu: 677 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 290, tpot_ms: 6.21, tokens_per_sec_per_gpu: 4401 }, + ttft_ms: 454, tpot_ms: 8.76, tokens_per_sec_per_gpu: 3059 }, ], }, { match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 4228, tpot_ms: 60.98, tokens_per_sec_per_gpu: 2025 }, + ttft_ms: 642, tpot_ms: 23.2, tokens_per_sec_per_gpu: 5222 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 4628, tpot_ms: 88.25, tokens_per_sec_per_gpu: 5787 }, + ttft_ms: 3147, tpot_ms: 64.0, tokens_per_sec_per_gpu: 8399 }, ], }, { match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 99949, tpot_ms: 67.46, tokens_per_sec_per_gpu: 8451 }, + ttft_ms: 104109, tpot_ms: 70.25, tokens_per_sec_per_gpu: 8345 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, - ttft_ms: 253310, tpot_ms: 66.11, tokens_per_sec_per_gpu: 8676 }, + ttft_ms: 273808, tpot_ms: 71.34, tokens_per_sec_per_gpu: 8156 }, ], }, { match: { hw: "b200", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 230, tpot_ms: 4.25, tokens_per_sec_per_gpu: 243 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 446, tpot_ms: 11.56, tokens_per_sec_per_gpu: 1165 }, + ], }, { match: { hw: "b200", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 2326, tpot_ms: 69.9, tokens_per_sec_per_gpu: 891 }, + ttft_ms: 1081, tpot_ms: 36.23, tokens_per_sec_per_gpu: 1696 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 7242, tpot_ms: 152.09, tokens_per_sec_per_gpu: 1728 }, + ttft_ms: 4330, tpot_ms: 97.59, tokens_per_sec_per_gpu: 2721 }, ], }, { match: { hw: "b200", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 107158, tpot_ms: 44.45, tokens_per_sec_per_gpu: 4169 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 265159, tpot_ms: 44.12, tokens_per_sec_per_gpu: 4252 }, + ], + }, + // ==================================================================== + // B200 + NVFP4 + // ==================================================================== + { + match: { hw: "b200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 308, tpot_ms: 2.88, tokens_per_sec_per_gpu: 682 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 466, tpot_ms: 8.67, tokens_per_sec_per_gpu: 3059 }, + ], + }, + { + match: { hw: "b200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 223, tpot_ms: 4.19, tokens_per_sec_per_gpu: 245 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 509, tpot_ms: 11.13, tokens_per_sec_per_gpu: 1210 }, + ], }, // ==================================================================== // B300 + FP4 // ==================================================================== { match: { hw: "b300", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 88, tpot_ms: 3.67, tokens_per_sec_per_gpu: 594 }, + ttft_ms: 191, tpot_ms: 2.87, tokens_per_sec_per_gpu: 720 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 266, tpot_ms: 6.06, tokens_per_sec_per_gpu: 4455 }, + ttft_ms: 360, tpot_ms: 8.05, tokens_per_sec_per_gpu: 3376 }, ], }, { match: { hw: "b300", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 2363, tpot_ms: 34.4, tokens_per_sec_per_gpu: 3618 }, + ttft_ms: 1317, tpot_ms: 33.78, tokens_per_sec_per_gpu: 3801 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 2812, tpot_ms: 51.65, tokens_per_sec_per_gpu: 9828 }, + ttft_ms: 2722, tpot_ms: 52.84, tokens_per_sec_per_gpu: 9773 }, ], }, { match: { hw: "b300", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 97028, tpot_ms: 65.09, tokens_per_sec_per_gpu: 8694 }, + ttft_ms: 89936, tpot_ms: 61.42, tokens_per_sec_per_gpu: 9336 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, - ttft_ms: 243335, tpot_ms: 63.98, tokens_per_sec_per_gpu: 8982 }, + ttft_ms: 238636, tpot_ms: 61.15, tokens_per_sec_per_gpu: 9432 }, ], }, { match: { hw: "b300", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 261, tpot_ms: 5.01, tokens_per_sec_per_gpu: 207 }, + ttft_ms: 258, tpot_ms: 4.2, tokens_per_sec_per_gpu: 243 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 364, tpot_ms: 11.37, tokens_per_sec_per_gpu: 1233 }, + ttft_ms: 460, tpot_ms: 10.97, tokens_per_sec_per_gpu: 1149 }, ], }, { match: { hw: "b300", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 1866, tpot_ms: 54.48, tokens_per_sec_per_gpu: 1251 }, + ttft_ms: 1868, tpot_ms: 42.19, tokens_per_sec_per_gpu: 1336 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 6325, tpot_ms: 123.95, tokens_per_sec_per_gpu: 2133 }, + ttft_ms: 2917, tpot_ms: 99.32, tokens_per_sec_per_gpu: 2669 }, ], }, { + // At conc 4096 the engine is saturated (running at its max batch), so extra requests + // queue — the high TTFT is queue wait, not compute; throughput is at its ceiling here. match: { hw: "b300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 103678, tpot_ms: 43.99, tokens_per_sec_per_gpu: 4203 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 257656, tpot_ms: 42.13, tokens_per_sec_per_gpu: 4400 }, + ], + }, + // ==================================================================== + // B300 + NVFP4 + // ==================================================================== + { + match: { hw: "b300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 187, tpot_ms: 2.83, tokens_per_sec_per_gpu: 729 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 410, tpot_ms: 7.68, tokens_per_sec_per_gpu: 3407 }, + ], + }, + { + match: { hw: "b300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 206, tpot_ms: 4.14, tokens_per_sec_per_gpu: 251 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 425, tpot_ms: 10.51, tokens_per_sec_per_gpu: 1256 }, + ], }, // ==================================================================== // GB200 + FP4 @@ -118,6 +186,7 @@ export const benchmarks = [ { match: { hw: "gb200", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, sglang_version: "0.5.12.post1", + latencyPercentile: "Mean", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, ttft_ms: 2560, tpot_ms: 39.71, tokens_per_sec_per_gpu: 3078 }, @@ -143,6 +212,7 @@ export const benchmarks = [ { match: { hw: "gb200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, sglang_version: "PR #25820", + latencyPercentile: "Mean", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, ttft_ms: 323.85, tpot_ms: 3.62, tokens_per_sec_per_gpu: 496 }, @@ -154,6 +224,7 @@ export const benchmarks = [ { match: { hw: "gb200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" }, sglang_version: "PR #25820", + latencyPercentile: "Mean", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, ttft_ms: 338.20, tpot_ms: 6.25, tokens_per_sec_per_gpu: 161 }, @@ -167,99 +238,118 @@ export const benchmarks = [ // ==================================================================== { match: { hw: "gb300", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.13.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 463, tpot_ms: 4.19, tokens_per_sec_per_gpu: 315 }, + ttft_ms: 434, tpot_ms: 3.72, tokens_per_sec_per_gpu: 513 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 436, tpot_ms: 8.93, tokens_per_sec_per_gpu: 3024 }, + ttft_ms: 735, tpot_ms: 9.95, tokens_per_sec_per_gpu: 2465 }, ], }, { match: { hw: "gb300", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 2671, tpot_ms: 45.88, tokens_per_sec_per_gpu: 2691 }, + ttft_ms: 1041, tpot_ms: 30.45, tokens_per_sec_per_gpu: 4022 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 4823, tpot_ms: 94.04, tokens_per_sec_per_gpu: 5733 }, + ttft_ms: 4291, tpot_ms: 85.9, tokens_per_sec_per_gpu: 6366 }, ], }, { match: { hw: "gb300", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 154868, tpot_ms: 104.84, tokens_per_sec_per_gpu: 5589 }, + ttft_ms: 137866, tpot_ms: 93.14, tokens_per_sec_per_gpu: 6338 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, - ttft_ms: 386489, tpot_ms: 103.37, tokens_per_sec_per_gpu: 5643 }, + ttft_ms: 364274, tpot_ms: 93.27, tokens_per_sec_per_gpu: 6246 }, ], }, { match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 317, tpot_ms: 4.49, tokens_per_sec_per_gpu: 441 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 501, tpot_ms: 14.54, tokens_per_sec_per_gpu: 1934 }, + ], }, { match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 1088, tpot_ms: 50.17, tokens_per_sec_per_gpu: 2455 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 8122, tpot_ms: 156.18, tokens_per_sec_per_gpu: 3429 }, + ], }, { match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 288182, tpot_ms: 185.19, tokens_per_sec_per_gpu: 2832 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 761128, tpot_ms: 188.23, tokens_per_sec_per_gpu: 2787 }, + ], }, // ==================================================================== // GB300 + NVFP4 // ==================================================================== { match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "PR #25820", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 361.72 , tpot_ms: 3.62, tokens_per_sec_per_gpu: 480 }, + ttft_ms: 430, tpot_ms: 3.51, tokens_per_sec_per_gpu: 537 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 422.96, tpot_ms: 8.19, tokens_per_sec_per_gpu: 3733 }, + ttft_ms: 734, tpot_ms: 10.59, tokens_per_sec_per_gpu: 2385 }, ], - accuracy: { gsm8k_pct: 96.44 }, }, { match: { hw: "gb300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "PR #25820", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 364.33, tpot_ms: 5.33, tokens_per_sec_per_gpu: 381 }, + ttft_ms: 321, tpot_ms: 4.61, tokens_per_sec_per_gpu: 440 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 694.58, tpot_ms: 16.03, tokens_per_sec_per_gpu: 1654 }, + ttft_ms: 631, tpot_ms: 14.25, tokens_per_sec_per_gpu: 1921 }, ], - accuracy: { gsm8k_pct: 96.36 }, }, // ==================================================================== // H200 + FP8 // ==================================================================== { match: { hw: "h200", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 204, tpot_ms: 3.38, tokens_per_sec_per_gpu: 612 }, + ttft_ms: 183, tpot_ms: 3.26, tokens_per_sec_per_gpu: 632 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 538, tpot_ms: 11.42, tokens_per_sec_per_gpu: 2376 }, + ttft_ms: 655, tpot_ms: 10.11, tokens_per_sec_per_gpu: 2752 }, ], }, { match: { hw: "h200", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, - ttft_ms: 738, tpot_ms: 36.27, tokens_per_sec_per_gpu: 3465 }, + ttft_ms: 880, tpot_ms: 40.63, tokens_per_sec_per_gpu: 3156 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, - ttft_ms: 39806, tpot_ms: 80.13, tokens_per_sec_per_gpu: 3537 }, + ttft_ms: 46563, tpot_ms: 89.82, tokens_per_sec_per_gpu: 3226 }, ], }, { match: { hw: "h200", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, - ttft_ms: 195293, tpot_ms: 130.35, tokens_per_sec_per_gpu: 4437 }, + ttft_ms: 217694, tpot_ms: 146.95, tokens_per_sec_per_gpu: 3975 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, - ttft_ms: 502615, tpot_ms: 130.31, tokens_per_sec_per_gpu: 4410 }, + ttft_ms: 576540, tpot_ms: 148.29, tokens_per_sec_per_gpu: 3920 }, ], }, { @@ -276,40 +366,100 @@ export const benchmarks = [ // ==================================================================== { match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, - sglang_version: "0.5.12.post1", + sglang_version: "0.5.15.post1", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, - ttft_ms: 193, tpot_ms: 3.38, tokens_per_sec_per_gpu: 603 }, + ttft_ms: 242, tpot_ms: 3.37, tokens_per_sec_per_gpu: 603 }, { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, - ttft_ms: 598, tpot_ms: 10.46, tokens_per_sec_per_gpu: 2772 }, + ttft_ms: 498, tpot_ms: 10.19, tokens_per_sec_per_gpu: 2636 }, ], }, { match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 864, tpot_ms: 34.12, tokens_per_sec_per_gpu: 3072 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 3222, tpot_ms: 116.3, tokens_per_sec_per_gpu: 3768 }, + ], }, { match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 193812, tpot_ms: 126.31, tokens_per_sec_per_gpu: 4503 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 499528, tpot_ms: 125.07, tokens_per_sec_per_gpu: 4546 }, + ], }, { match: { hw: "h200", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 634, tpot_ms: 5.65, tokens_per_sec_per_gpu: 170 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 1727, tpot_ms: 23.12, tokens_per_sec_per_gpu: 559 }, + ], }, { + // Capacity-bound on 8xH200 for the 1.6T model: KV fits only ~15 concurrent requests, so + // tok/s/GPU is pinned (~535-572) from conc 64 through the ht conc-4096 cell and the excess + // concurrency just queues — P50 TTFT climbs to ~46s here and minutes at higher conc. The + // throughput numbers are real but reflect that ceiling, not linear scaling. match: { hw: "h200", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 41506, tpot_ms: 26.14, tokens_per_sec_per_gpu: 589 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 209586, tpot_ms: 28.23, tokens_per_sec_per_gpu: 591 }, + ], }, { match: { hw: "h200", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.15.post1", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 889185, tpot_ms: 66.39, tokens_per_sec_per_gpu: 594 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 1833386, tpot_ms: 65.86, tokens_per_sec_per_gpu: 601 }, + ], }, // ==================================================================== // H100 + FP4 // ==================================================================== { match: { hw: "h100", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 205, tpot_ms: 3.19, tokens_per_sec_per_gpu: 319 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 469, tpot_ms: 8.48, tokens_per_sec_per_gpu: 1539 }, + ], }, { match: { hw: "h100", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 }, + ttft_ms: 726, tpot_ms: 23.11, tokens_per_sec_per_gpu: 2306 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 }, + ttft_ms: 35793, tpot_ms: 48.46, tokens_per_sec_per_gpu: 2416 }, + ], }, { match: { hw: "h100", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "0.5.15", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 }, + ttft_ms: 209393, tpot_ms: 65.31, tokens_per_sec_per_gpu: 2252 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 }, + ttft_ms: 476764, tpot_ms: 66.0, tokens_per_sec_per_gpu: 2248 }, + ], }, { match: { hw: "h100", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "multi-2" }, diff --git a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx index df4b109ef..3d32ed2ef 100644 --- a/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx +++ b/docs_new/src/snippets/configs/deepseek-ai/deepseek-v4.jsx @@ -4,7 +4,7 @@ export const config = { modelName: "DeepSeek-V4", - latencyPercentile: "Mean", // temporary; re-measure to P50 + latencyPercentile: "P50", supportedHardware: [ "h100", "h200", "b200", "b300", "gb200", "gb300", @@ -80,7 +80,7 @@ export const config = { --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} \\ --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ - --warmup-requests 64`, + --warmup-requests 64 --flush-cache`, accuracy: { gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval @@ -1276,7 +1276,7 @@ sgl-eval run aime25 \\ "--speculative-num-steps 3", "--speculative-eagle-topk 1", "--speculative-num-draft-tokens 4", - "--mem-fraction-static 0.83", + "--mem-fraction-static 0.90", "--host {{HOST_IP}}", "--port {{PORT}}", ],