docs(cookbook): re-benchmark DeepSeek-V4 on sglang 0.5.15 (#31363)
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Zijie Xia <zijie.xia@radixark.ai>
This commit is contained in:
Douglas Yang
co-authored by
Claude Opus 4.8
Zijie Xia
Claude Opus 4.8
Zijie Xia
parent
9057db9417
commit
4a55fdba0b
@@ -25,6 +25,8 @@
|
||||
// required whenever benchmarks carry accuracy data
|
||||
// latencyPercentile optional, TEMPORARY — "Mean" | "P50" (default "P50"); the
|
||||
// percentile the TTFT/TPOT values are, shown as "TTFT (<pct>)".
|
||||
// A benchmarks entry may carry its own latencyPercentile to
|
||||
// override the page value per cell (entry → config → "P50").
|
||||
// Legacy "Mean" data is being re-measured to P50; drop once done
|
||||
// multiNodeHints optional — {[hwId]: string[]} prepended as `# ...` lines
|
||||
// dockerImages optional — `docker run` image, keyed by `hw|quant`
|
||||
@@ -577,7 +579,7 @@ export const Deployment = ({ config, benchmarks }) => {
|
||||
const renderBenchmarkCard = (entry) => {
|
||||
// [key, label, unit, compute?]. Optional compute(measurement) supplies
|
||||
// derived metrics (preferred over measurement[key] when present).
|
||||
const pct = config.latencyPercentile || "P50";
|
||||
const pct = (entry && entry.latencyPercentile) || config.latencyPercentile || "P50";
|
||||
const SPEED_LABELS = [
|
||||
["ttft_ms", `TTFT (${pct})`, "ms"],
|
||||
["tpot_ms", `TPOT (${pct})`, "ms"],
|
||||
|
||||
@@ -1,113 +1,181 @@
|
||||
// DeepSeek-V4 per-cell benchmark numbers, keyed by the same `match` tuple as
|
||||
// deepseek-v4.jsx cells. See _deployment.jsx for the speed/accuracy schema.
|
||||
// Measured on sglang v0.5.12.post1.
|
||||
// tokens_per_sec_per_gpu is total (input+output) tok/s/GPU: fp4/fp8 = measured
|
||||
// output/GPU × (isl+osl)/osl; nvfp4 was measured as total already.
|
||||
// Measured on sglang v0.5.15 / v0.5.15.post1 (per-cell sglang_version).
|
||||
// tokens_per_sec_per_gpu is total (input+output) tok/s/GPU = output/GPU × (isl+osl)/osl.
|
||||
export const benchmarks = [
|
||||
// ====================================================================
|
||||
// B200 + FP4
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 87, tpot_ms: 3.68, tokens_per_sec_per_gpu: 585 },
|
||||
ttft_ms: 302, tpot_ms: 2.91, tokens_per_sec_per_gpu: 677 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 290, tpot_ms: 6.21, tokens_per_sec_per_gpu: 4401 },
|
||||
ttft_ms: 454, tpot_ms: 8.76, tokens_per_sec_per_gpu: 3059 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 4228, tpot_ms: 60.98, tokens_per_sec_per_gpu: 2025 },
|
||||
ttft_ms: 642, tpot_ms: 23.2, tokens_per_sec_per_gpu: 5222 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 4628, tpot_ms: 88.25, tokens_per_sec_per_gpu: 5787 },
|
||||
ttft_ms: 3147, tpot_ms: 64.0, tokens_per_sec_per_gpu: 8399 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 99949, tpot_ms: 67.46, tokens_per_sec_per_gpu: 8451 },
|
||||
ttft_ms: 104109, tpot_ms: 70.25, tokens_per_sec_per_gpu: 8345 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 253310, tpot_ms: 66.11, tokens_per_sec_per_gpu: 8676 },
|
||||
ttft_ms: 273808, tpot_ms: 71.34, tokens_per_sec_per_gpu: 8156 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 230, tpot_ms: 4.25, tokens_per_sec_per_gpu: 243 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 446, tpot_ms: 11.56, tokens_per_sec_per_gpu: 1165 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 2326, tpot_ms: 69.9, tokens_per_sec_per_gpu: 891 },
|
||||
ttft_ms: 1081, tpot_ms: 36.23, tokens_per_sec_per_gpu: 1696 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 7242, tpot_ms: 152.09, tokens_per_sec_per_gpu: 1728 },
|
||||
ttft_ms: 4330, tpot_ms: 97.59, tokens_per_sec_per_gpu: 2721 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 107158, tpot_ms: 44.45, tokens_per_sec_per_gpu: 4169 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 265159, tpot_ms: 44.12, tokens_per_sec_per_gpu: 4252 },
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// B200 + NVFP4
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "b200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 308, tpot_ms: 2.88, tokens_per_sec_per_gpu: 682 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 466, tpot_ms: 8.67, tokens_per_sec_per_gpu: 3059 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 223, tpot_ms: 4.19, tokens_per_sec_per_gpu: 245 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 509, tpot_ms: 11.13, tokens_per_sec_per_gpu: 1210 },
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// B300 + FP4
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "b300", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 88, tpot_ms: 3.67, tokens_per_sec_per_gpu: 594 },
|
||||
ttft_ms: 191, tpot_ms: 2.87, tokens_per_sec_per_gpu: 720 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 266, tpot_ms: 6.06, tokens_per_sec_per_gpu: 4455 },
|
||||
ttft_ms: 360, tpot_ms: 8.05, tokens_per_sec_per_gpu: 3376 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b300", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 2363, tpot_ms: 34.4, tokens_per_sec_per_gpu: 3618 },
|
||||
ttft_ms: 1317, tpot_ms: 33.78, tokens_per_sec_per_gpu: 3801 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 2812, tpot_ms: 51.65, tokens_per_sec_per_gpu: 9828 },
|
||||
ttft_ms: 2722, tpot_ms: 52.84, tokens_per_sec_per_gpu: 9773 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b300", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 97028, tpot_ms: 65.09, tokens_per_sec_per_gpu: 8694 },
|
||||
ttft_ms: 89936, tpot_ms: 61.42, tokens_per_sec_per_gpu: 9336 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 243335, tpot_ms: 63.98, tokens_per_sec_per_gpu: 8982 },
|
||||
ttft_ms: 238636, tpot_ms: 61.15, tokens_per_sec_per_gpu: 9432 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b300", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 261, tpot_ms: 5.01, tokens_per_sec_per_gpu: 207 },
|
||||
ttft_ms: 258, tpot_ms: 4.2, tokens_per_sec_per_gpu: 243 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 364, tpot_ms: 11.37, tokens_per_sec_per_gpu: 1233 },
|
||||
ttft_ms: 460, tpot_ms: 10.97, tokens_per_sec_per_gpu: 1149 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b300", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 1866, tpot_ms: 54.48, tokens_per_sec_per_gpu: 1251 },
|
||||
ttft_ms: 1868, tpot_ms: 42.19, tokens_per_sec_per_gpu: 1336 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 6325, tpot_ms: 123.95, tokens_per_sec_per_gpu: 2133 },
|
||||
ttft_ms: 2917, tpot_ms: 99.32, tokens_per_sec_per_gpu: 2669 },
|
||||
],
|
||||
},
|
||||
{
|
||||
// At conc 4096 the engine is saturated (running at its max batch), so extra requests
|
||||
// queue — the high TTFT is queue wait, not compute; throughput is at its ceiling here.
|
||||
match: { hw: "b300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 103678, tpot_ms: 43.99, tokens_per_sec_per_gpu: 4203 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 257656, tpot_ms: 42.13, tokens_per_sec_per_gpu: 4400 },
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// B300 + NVFP4
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "b300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 187, tpot_ms: 2.83, tokens_per_sec_per_gpu: 729 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 410, tpot_ms: 7.68, tokens_per_sec_per_gpu: 3407 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 206, tpot_ms: 4.14, tokens_per_sec_per_gpu: 251 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 425, tpot_ms: 10.51, tokens_per_sec_per_gpu: 1256 },
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// GB200 + FP4
|
||||
@@ -118,6 +186,7 @@ export const benchmarks = [
|
||||
{
|
||||
match: { hw: "gb200", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
latencyPercentile: "Mean",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 2560, tpot_ms: 39.71, tokens_per_sec_per_gpu: 3078 },
|
||||
@@ -143,6 +212,7 @@ export const benchmarks = [
|
||||
{
|
||||
match: { hw: "gb200", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "PR #25820",
|
||||
latencyPercentile: "Mean",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 323.85, tpot_ms: 3.62, tokens_per_sec_per_gpu: 496 },
|
||||
@@ -154,6 +224,7 @@ export const benchmarks = [
|
||||
{
|
||||
match: { hw: "gb200", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "multi-2" },
|
||||
sglang_version: "PR #25820",
|
||||
latencyPercentile: "Mean",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 338.20, tpot_ms: 6.25, tokens_per_sec_per_gpu: 161 },
|
||||
@@ -167,99 +238,118 @@ export const benchmarks = [
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "gb300", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.13.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 463, tpot_ms: 4.19, tokens_per_sec_per_gpu: 315 },
|
||||
ttft_ms: 434, tpot_ms: 3.72, tokens_per_sec_per_gpu: 513 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 436, tpot_ms: 8.93, tokens_per_sec_per_gpu: 3024 },
|
||||
ttft_ms: 735, tpot_ms: 9.95, tokens_per_sec_per_gpu: 2465 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 2671, tpot_ms: 45.88, tokens_per_sec_per_gpu: 2691 },
|
||||
ttft_ms: 1041, tpot_ms: 30.45, tokens_per_sec_per_gpu: 4022 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 4823, tpot_ms: 94.04, tokens_per_sec_per_gpu: 5733 },
|
||||
ttft_ms: 4291, tpot_ms: 85.9, tokens_per_sec_per_gpu: 6366 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 154868, tpot_ms: 104.84, tokens_per_sec_per_gpu: 5589 },
|
||||
ttft_ms: 137866, tpot_ms: 93.14, tokens_per_sec_per_gpu: 6338 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 386489, tpot_ms: 103.37, tokens_per_sec_per_gpu: 5643 },
|
||||
ttft_ms: 364274, tpot_ms: 93.27, tokens_per_sec_per_gpu: 6246 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 317, tpot_ms: 4.49, tokens_per_sec_per_gpu: 441 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 501, tpot_ms: 14.54, tokens_per_sec_per_gpu: 1934 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 1088, tpot_ms: 50.17, tokens_per_sec_per_gpu: 2455 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 8122, tpot_ms: 156.18, tokens_per_sec_per_gpu: 3429 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 288182, tpot_ms: 185.19, tokens_per_sec_per_gpu: 2832 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 761128, tpot_ms: 188.23, tokens_per_sec_per_gpu: 2787 },
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// GB300 + NVFP4
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "gb300", variant: "flash", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "PR #25820",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 361.72 , tpot_ms: 3.62, tokens_per_sec_per_gpu: 480 },
|
||||
ttft_ms: 430, tpot_ms: 3.51, tokens_per_sec_per_gpu: 537 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 422.96, tpot_ms: 8.19, tokens_per_sec_per_gpu: 3733 },
|
||||
ttft_ms: 734, tpot_ms: 10.59, tokens_per_sec_per_gpu: 2385 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.44 },
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "pro", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "PR #25820",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 364.33, tpot_ms: 5.33, tokens_per_sec_per_gpu: 381 },
|
||||
ttft_ms: 321, tpot_ms: 4.61, tokens_per_sec_per_gpu: 440 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 694.58, tpot_ms: 16.03, tokens_per_sec_per_gpu: 1654 },
|
||||
ttft_ms: 631, tpot_ms: 14.25, tokens_per_sec_per_gpu: 1921 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 96.36 },
|
||||
},
|
||||
// ====================================================================
|
||||
// H200 + FP8
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "fp8", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 204, tpot_ms: 3.38, tokens_per_sec_per_gpu: 612 },
|
||||
ttft_ms: 183, tpot_ms: 3.26, tokens_per_sec_per_gpu: 632 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 538, tpot_ms: 11.42, tokens_per_sec_per_gpu: 2376 },
|
||||
ttft_ms: 655, tpot_ms: 10.11, tokens_per_sec_per_gpu: 2752 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "fp8", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 738, tpot_ms: 36.27, tokens_per_sec_per_gpu: 3465 },
|
||||
ttft_ms: 880, tpot_ms: 40.63, tokens_per_sec_per_gpu: 3156 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 39806, tpot_ms: 80.13, tokens_per_sec_per_gpu: 3537 },
|
||||
ttft_ms: 46563, tpot_ms: 89.82, tokens_per_sec_per_gpu: 3226 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 195293, tpot_ms: 130.35, tokens_per_sec_per_gpu: 4437 },
|
||||
ttft_ms: 217694, tpot_ms: 146.95, tokens_per_sec_per_gpu: 3975 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 502615, tpot_ms: 130.31, tokens_per_sec_per_gpu: 4410 },
|
||||
ttft_ms: 576540, tpot_ms: 148.29, tokens_per_sec_per_gpu: 3920 },
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -276,40 +366,100 @@ export const benchmarks = [
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.12.post1",
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 193, tpot_ms: 3.38, tokens_per_sec_per_gpu: 603 },
|
||||
ttft_ms: 242, tpot_ms: 3.37, tokens_per_sec_per_gpu: 603 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 598, tpot_ms: 10.46, tokens_per_sec_per_gpu: 2772 },
|
||||
ttft_ms: 498, tpot_ms: 10.19, tokens_per_sec_per_gpu: 2636 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 864, tpot_ms: 34.12, tokens_per_sec_per_gpu: 3072 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 3222, tpot_ms: 116.3, tokens_per_sec_per_gpu: 3768 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 193812, tpot_ms: 126.31, tokens_per_sec_per_gpu: 4503 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 499528, tpot_ms: 125.07, tokens_per_sec_per_gpu: 4546 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 634, tpot_ms: 5.65, tokens_per_sec_per_gpu: 170 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 1727, tpot_ms: 23.12, tokens_per_sec_per_gpu: 559 },
|
||||
],
|
||||
},
|
||||
{
|
||||
// Capacity-bound on 8xH200 for the 1.6T model: KV fits only ~15 concurrent requests, so
|
||||
// tok/s/GPU is pinned (~535-572) from conc 64 through the ht conc-4096 cell and the excess
|
||||
// concurrency just queues — P50 TTFT climbs to ~46s here and minutes at higher conc. The
|
||||
// throughput numbers are real but reflect that ceiling, not linear scaling.
|
||||
match: { hw: "h200", variant: "pro", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 41506, tpot_ms: 26.14, tokens_per_sec_per_gpu: 589 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 209586, tpot_ms: 28.23, tokens_per_sec_per_gpu: 591 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "pro", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.15.post1",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 889185, tpot_ms: 66.39, tokens_per_sec_per_gpu: 594 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 1833386, tpot_ms: 65.86, tokens_per_sec_per_gpu: 601 },
|
||||
],
|
||||
},
|
||||
// ====================================================================
|
||||
// H100 + FP4
|
||||
// ====================================================================
|
||||
{
|
||||
match: { hw: "h100", variant: "flash", quant: "fp4", strategy: "low-latency", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 205, tpot_ms: 3.19, tokens_per_sec_per_gpu: 319 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 469, tpot_ms: 8.48, tokens_per_sec_per_gpu: 1539 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "flash", quant: "fp4", strategy: "balanced", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 64 },
|
||||
ttft_ms: 726, tpot_ms: 23.11, tokens_per_sec_per_gpu: 2306 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 256 },
|
||||
ttft_ms: 35793, tpot_ms: 48.46, tokens_per_sec_per_gpu: 2416 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "flash", quant: "fp4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "0.5.15",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1024 },
|
||||
ttft_ms: 209393, tpot_ms: 65.31, tokens_per_sec_per_gpu: 2252 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 4096 },
|
||||
ttft_ms: 476764, tpot_ms: 66.0, tokens_per_sec_per_gpu: 2248 },
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "pro", quant: "fp4", strategy: "low-latency", nodes: "multi-2" },
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
export const config = {
|
||||
modelName: "DeepSeek-V4",
|
||||
|
||||
latencyPercentile: "Mean", // temporary; re-measure to P50
|
||||
latencyPercentile: "P50",
|
||||
|
||||
supportedHardware: [
|
||||
"h100", "h200", "b200", "b300", "gb200", "gb300",
|
||||
@@ -80,7 +80,7 @@ export const config = {
|
||||
--dataset-name {{DATASET}} \\
|
||||
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
|
||||
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
|
||||
--warmup-requests 64`,
|
||||
--warmup-requests 64 --flush-cache`,
|
||||
accuracy: {
|
||||
gsm8k_pct:
|
||||
`# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
|
||||
@@ -1276,7 +1276,7 @@ sgl-eval run aime25 \\
|
||||
"--speculative-num-steps 3",
|
||||
"--speculative-eagle-topk 1",
|
||||
"--speculative-num-draft-tokens 4",
|
||||
"--mem-fraction-static 0.83",
|
||||
"--mem-fraction-static 0.90",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user