minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Doc] Cookbook Laguna-XS-2.1: add AIME25 accuracy (B300 + GB300) (#29974)

Browse Source
This commit is contained in:
Jimmy Shong
2026-07-02 13:15:01 -07:00
committed by GitHub
parent f19246e59a
commit 85e71b7e13
2 changed files with 65 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx
+51 -22
View File
@@ -65,6 +65,35 @@
// Every cell at parity with its tp4-GB300 and H200 references; NVFP4 needs NO escape
// (group_size=16 divides the 64-wide tp8 shard — unlike FP8 [128,128] / INT4 gs=128).
//
// REAL AIME25 (sgl-eval `run aime25`, 30 problems x 16 repeats, temperature 1.0, top-p 0.95,
// max-tokens 64000, 128 threads; value shipped = pass@1[avg-of-16], SEM ~1.4pt/cell at 480
// samples). Thinking ENABLED by serving with a copy of the model's chat template whose
// enable_thinking default is flipped to true — sgl-eval's --thinking sets the generic
// 'thinking' key, which Laguna's template ignores (see Configuration Tips: Thinking).
// Run @ main 0543246184.
//
// B300 (same 2x(4xGB300) tp8/MNNVL topology + shard math as the GSM8K B300 numbers):
// high-throughput: BF16 65.21 | FP8 61.67 | NVFP4 57.92 | INT4 63.54
// low-latency: BF16 65.62 | FP8 62.50 | NVFP4 60.21 | INT4 62.92
// GB300 (4-GPU single node, tp 4):
// high-throughput: BF16 62.50 | FP8 63.12 | NVFP4 60.00 | INT4 64.79
// low-latency: BF16 65.83 | FP8 63.12 | NVFP4 60.00 | INT4 61.04
//
// H200 (8-GPU HGX; bf16 tp8, fp8/int4 tp8+ep8 — same recipes as GSM8K):
// high-throughput: BF16 63.96 | FP8 64.79 | INT4 63.33
// low-latency: BF16 65.00 | FP8 62.50 | INT4 64.17
//
// DFlash accuracy-neutral on AIME25 too (|dense-spec| <= 2.7pt ~ 1-2 SEM); accept-len ~2.9
// on long thinking traces (vs ~4 on greedy GSM8K). NVFP4 is the weakest quant on AIME25
// (~4-5 SEM below BF16) while being the strongest on GSM8K — quant rankings are
// benchmark-dependent. Truncation ~0% at the 64k cap.
// H200 tp8+ep8 vs tp4/tp8-plain reference (same eval shape, different sglang session):
// BF16 dense 65.83->63.96 (1.25 SEM), FP8 dense 61.67->64.79 (2.50 SEM, higher not lower —
// FP8-dense alone now spans ~3pt across 4 independent full-set-equivalent measurements this
// week, so this is ordinary AIME variance for a 30x16 eval, not an EP8 effect), all other
// cells <=1.05 SEM. pass@16/majority@16 (single 30-item proportions, ~6-8pt SE) all <1 SEM;
// full detail in the day-0 support log, not reproduced here.
export const benchmarks = [
// ===== H200 (8-GPU HGX; bf16 tp 8, fp8/int4 tp8+ep8) — ✅ REAL, full GSM8K =====
{
@@ -72,7 +101,7 @@ export const benchmarks = [
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 76.12 },
accuracy: { gsm8k_pct: 76.12, aime25_pct: 63.96 },
},
{
// ✅ REAL — 8×H200, BF16 + DFlash (matched bf16 draft), tp8, fa3. Accept-len 3.05
@@ -80,7 +109,7 @@ export const benchmarks = [
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 75.97 },
accuracy: { gsm8k_pct: 75.97, aime25_pct: 65.00 },
},
{
// ✅ REAL — 8×H200, FP8 dense, tp8+ep8+SGLANG_SHARED_EXPERT_TP1=1 (plain tp8 impossible:
@@ -88,7 +117,7 @@ export const benchmarks = [
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 73.54 },
accuracy: { gsm8k_pct: 73.54, aime25_pct: 64.79 },
},
{
// ✅ REAL — 8×H200, FP8 + DFlash (matched fp8-calibrated draft), tp8+ep8+flag, fa3.
@@ -96,7 +125,7 @@ export const benchmarks = [
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 74.53 },
accuracy: { gsm8k_pct: 74.53, aime25_pct: 62.50 },
},
{
// ✅ REAL — 8×H200, INT4 dense (mixed 4/8-bit MoE, needs #29761), tp8+ep8 (plain tp8
@@ -104,7 +133,7 @@ export const benchmarks = [
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 67.02 },
accuracy: { gsm8k_pct: 67.02, aime25_pct: 63.33 },
},
{
// ✅ REAL — 8×H200, INT4 + DFlash (matched int4-calibrated draft), tp8+ep8, fa3.
@@ -113,7 +142,7 @@ export const benchmarks = [
match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 66.57 },
accuracy: { gsm8k_pct: 66.57, aime25_pct: 64.17 },
},
// ===== B300 (8-GPU HGX; bf16/nvfp4 tp 8, fp8/int4 tp8+ep8) — REAL, full GSM8K =====
@@ -123,56 +152,56 @@ export const benchmarks = [
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 75.59 },
accuracy: { gsm8k_pct: 75.59, aime25_pct: 65.21 },
},
{
// REAL — BF16 + DFlash (matched bf16 draft), tp8, trtllm_mha. Accept-len 4.08.
match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 75.36 },
accuracy: { gsm8k_pct: 75.36, aime25_pct: 65.62 },
},
{
// REAL — FP8 dense, tp8+ep8+SGLANG_SHARED_EXPERT_TP1=1 (plain tp8 impossible: block-FP8 scale granularity).
match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 71.19 },
accuracy: { gsm8k_pct: 71.19, aime25_pct: 61.67 },
},
{
// REAL — FP8 + DFlash (matched fp8-calibrated draft), tp8+ep8+flag, trtllm_mha. Accept-len 4.05.
match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 71.87 },
accuracy: { gsm8k_pct: 71.87, aime25_pct: 62.50 },
},
{
// REAL — NVFP4 dense, tp8 — NO escape needed (group_size=16 shards 8-way cleanly).
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 78.01 },
accuracy: { gsm8k_pct: 78.01, aime25_pct: 57.92 },
},
{
// REAL — NVFP4 + DFlash (matched nvfp4-calibrated draft), tp8, trtllm_mha. Accept-len 4.04.
match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 77.79 },
accuracy: { gsm8k_pct: 77.79, aime25_pct: 60.21 },
},
{
// REAL — INT4 dense (mixed 4/8-bit MoE), tp8+ep8 (plain tp8 impossible: Marlin gs=128 'scales is not contiguous', same signature as H200).
match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 67.25 },
accuracy: { gsm8k_pct: 67.25, aime25_pct: 63.54 },
},
{
// REAL — INT4 + DFlash (matched int4-calibrated draft), tp8+ep8, trtllm_mha. Accept-len 4.01.
match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
accuracy: { gsm8k_pct: 66.72 },
accuracy: { gsm8k_pct: 66.72, aime25_pct: 62.92 },
},
// ===== GB300 (4-GPU single node, tp 4) — ✅ REAL, full GSM8K =====
@@ -181,55 +210,55 @@ export const benchmarks = [
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 75.66 },
accuracy: { gsm8k_pct: 75.66, aime25_pct: 62.50 },
},
{
// ✅ REAL — 4×GB300, BF16 + DFlash (matched bf16 draft), tp4, trtllm_mha. Accept-len 4.17.
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 76.19 },
accuracy: { gsm8k_pct: 76.19, aime25_pct: 65.83 },
},
{
// ✅ REAL — 4×GB300, FP8 dense, tp4, backend auto→trtllm_mha.
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 71.87 },
accuracy: { gsm8k_pct: 71.87, aime25_pct: 63.12 },
},
{
// ✅ REAL — 4×GB300, FP8 + DFlash (matched fp8-calibrated draft), tp4, trtllm_mha. Accept-len 4.05.
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 72.02 },
accuracy: { gsm8k_pct: 72.02, aime25_pct: 63.12 },
},
{
// ✅ REAL — 4×GB300, NVFP4 dense, tp4, backend auto→trtllm_mha.
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 78.39 },
accuracy: { gsm8k_pct: 78.39, aime25_pct: 60.00 },
},
{
// ✅ REAL — 4×GB300, NVFP4 + DFlash (matched nvfp4-calibrated draft), tp4, trtllm_mha. Accept-len 4.02.
match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 74.53 },
accuracy: { gsm8k_pct: 74.53, aime25_pct: 60.00 },
},
{
// ✅ REAL — 4×GB300, INT4 dense (mixed 4/8-bit MoE, needs #29761), tp4, backend auto→trtllm_mha.
match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 66.79 },
accuracy: { gsm8k_pct: 66.79, aime25_pct: 64.79 },
},
{
// ✅ REAL — 4×GB300, INT4 + DFlash (matched int4-calibrated draft), tp4, trtllm_mha. Accept-len 3.80.
match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" },
verified: true,
sglang_version: "PR #29446 + #29761 (both merged to main)",
accuracy: { gsm8k_pct: 67.02 },
accuracy: { gsm8k_pct: 67.02, aime25_pct: 61.04 },
},
];
docs_new/src/snippets/configs/poolside/laguna-xs21.jsx
+14 -1
View File
@@ -115,6 +115,18 @@ export const config = {
`# pip install git+https://github.com/sgl-project/sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 128`,
// sgl-eval's --thinking sets the generic 'thinking' key, which Laguna's template
// ignores (it gates on enable_thinking — see Configuration Tips: Thinking). To run
// AIME25 with thinking, serve with a copy of the model's chat template whose
// enable_thinking default is flipped to true, passed via --chat-template.
aime25_pct:
`# pip install git+https://github.com/sgl-project/sgl-eval
# Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking).
sgl-eval run aime25 \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--n-repeats 16 --max-tokens 64000 \\
--temperature 1.0 --top-p 0.95 --thinking \\
--num-threads 128`,
},
numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 },
@@ -122,11 +134,12 @@ sgl-eval run gsm8k \\
// No variant-wide accuracy default; real numbers are per-cell in laguna-xs21-benchmarks.jsx.
defaultAccuracy: {
default: { gsm8k_pct: null },
default: { gsm8k_pct: null, aime25_pct: null },
},
accuracyLabels: [
["gsm8k_pct", "GSM8K", "%"],
["aime25_pct", "AIME25", "%"],
],
// Dedicated image built for this cookbook page (PR #29446 + #29761 preinstalled on cu13).