From 85e71b7e13be31fedd45586d76374f6eac905388 Mon Sep 17 00:00:00 2001
From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com>
Date: Thu, 2 Jul 2026 13:15:01 -0700
Subject: [PATCH] [Doc] Cookbook Laguna-XS-2.1: add AIME25 accuracy (B300 +
 GB300) (#29974)

---
 .../poolside/laguna-xs21-benchmarks.jsx       | 73 +++++++++++++------
 .../snippets/configs/poolside/laguna-xs21.jsx | 15 +++-
 2 files changed, 65 insertions(+), 23 deletions(-)

diff --git a/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx
index c4b9e56be..9b5077887 100644
--- a/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx
+++ b/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx
@@ -65,6 +65,35 @@
 //   Every cell at parity with its tp4-GB300 and H200 references; NVFP4 needs NO escape
 //   (group_size=16 divides the 64-wide tp8 shard — unlike FP8 [128,128] / INT4 gs=128).
 
+//
+// REAL AIME25 (sgl-eval `run aime25`, 30 problems x 16 repeats, temperature 1.0, top-p 0.95,
+// max-tokens 64000, 128 threads; value shipped = pass@1[avg-of-16], SEM ~1.4pt/cell at 480
+// samples). Thinking ENABLED by serving with a copy of the model's chat template whose
+// enable_thinking default is flipped to true — sgl-eval's --thinking sets the generic
+// 'thinking' key, which Laguna's template ignores (see Configuration Tips: Thinking).
+// Run @ main 0543246184.
+//
+//   B300 (same 2x(4xGB300) tp8/MNNVL topology + shard math as the GSM8K B300 numbers):
+//     high-throughput: BF16 65.21 | FP8 61.67 | NVFP4 57.92 | INT4 63.54
+//     low-latency:     BF16 65.62 | FP8 62.50 | NVFP4 60.21 | INT4 62.92
+//   GB300 (4-GPU single node, tp 4):
+//     high-throughput: BF16 62.50 | FP8 63.12 | NVFP4 60.00 | INT4 64.79
+//     low-latency:     BF16 65.83 | FP8 63.12 | NVFP4 60.00 | INT4 61.04
+//
+//   H200 (8-GPU HGX; bf16 tp8, fp8/int4 tp8+ep8 — same recipes as GSM8K):
+//     high-throughput: BF16 63.96 | FP8 64.79 | INT4 63.33
+//     low-latency:     BF16 65.00 | FP8 62.50 | INT4 64.17
+//
+//   DFlash accuracy-neutral on AIME25 too (|dense-spec| <= 2.7pt ~ 1-2 SEM); accept-len ~2.9
+//   on long thinking traces (vs ~4 on greedy GSM8K). NVFP4 is the weakest quant on AIME25
+//   (~4-5 SEM below BF16) while being the strongest on GSM8K — quant rankings are
+//   benchmark-dependent. Truncation ~0% at the 64k cap.
+//   H200 tp8+ep8 vs tp4/tp8-plain reference (same eval shape, different sglang session):
+//   BF16 dense 65.83->63.96 (1.25 SEM), FP8 dense 61.67->64.79 (2.50 SEM, higher not lower —
+//   FP8-dense alone now spans ~3pt across 4 independent full-set-equivalent measurements this
+//   week, so this is ordinary AIME variance for a 30x16 eval, not an EP8 effect), all other
+//   cells <=1.05 SEM. pass@16/majority@16 (single 30-item proportions, ~6-8pt SE) all <1 SEM;
+//   full detail in the day-0 support log, not reproduced here.
 export const benchmarks = [
   // ===== H200 (8-GPU HGX; bf16 tp 8, fp8/int4 tp8+ep8) — ✅ REAL, full GSM8K =====
   {
@@ -72,7 +101,7 @@ export const benchmarks = [
     match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 76.12 },
+    accuracy: { gsm8k_pct: 76.12, aime25_pct: 63.96 },
   },
   {
     // ✅ REAL — 8×H200, BF16 + DFlash (matched bf16 draft), tp8, fa3. Accept-len 3.05
@@ -80,7 +109,7 @@ export const benchmarks = [
     match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 75.97 },
+    accuracy: { gsm8k_pct: 75.97, aime25_pct: 65.00 },
   },
   {
     // ✅ REAL — 8×H200, FP8 dense, tp8+ep8+SGLANG_SHARED_EXPERT_TP1=1 (plain tp8 impossible:
@@ -88,7 +117,7 @@ export const benchmarks = [
     match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 73.54 },
+    accuracy: { gsm8k_pct: 73.54, aime25_pct: 64.79 },
   },
   {
     // ✅ REAL — 8×H200, FP8 + DFlash (matched fp8-calibrated draft), tp8+ep8+flag, fa3.
@@ -96,7 +125,7 @@ export const benchmarks = [
     match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 74.53 },
+    accuracy: { gsm8k_pct: 74.53, aime25_pct: 62.50 },
   },
   {
     // ✅ REAL — 8×H200, INT4 dense (mixed 4/8-bit MoE, needs #29761), tp8+ep8 (plain tp8
@@ -104,7 +133,7 @@ export const benchmarks = [
     match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 67.02 },
+    accuracy: { gsm8k_pct: 67.02, aime25_pct: 63.33 },
   },
   {
     // ✅ REAL — 8×H200, INT4 + DFlash (matched int4-calibrated draft), tp8+ep8, fa3.
@@ -113,7 +142,7 @@ export const benchmarks = [
     match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 66.57 },
+    accuracy: { gsm8k_pct: 66.57, aime25_pct: 64.17 },
   },
 
   // ===== B300 (8-GPU HGX; bf16/nvfp4 tp 8, fp8/int4 tp8+ep8) — REAL, full GSM8K =====
@@ -123,56 +152,56 @@ export const benchmarks = [
     match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 75.59 },
+    accuracy: { gsm8k_pct: 75.59, aime25_pct: 65.21 },
   },
   {
     // REAL — BF16 + DFlash (matched bf16 draft), tp8, trtllm_mha. Accept-len 4.08.
     match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 75.36 },
+    accuracy: { gsm8k_pct: 75.36, aime25_pct: 65.62 },
   },
   {
     // REAL — FP8 dense, tp8+ep8+SGLANG_SHARED_EXPERT_TP1=1 (plain tp8 impossible: block-FP8 scale granularity).
     match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 71.19 },
+    accuracy: { gsm8k_pct: 71.19, aime25_pct: 61.67 },
   },
   {
     // REAL — FP8 + DFlash (matched fp8-calibrated draft), tp8+ep8+flag, trtllm_mha. Accept-len 4.05.
     match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 71.87 },
+    accuracy: { gsm8k_pct: 71.87, aime25_pct: 62.50 },
   },
   {
     // REAL — NVFP4 dense, tp8 — NO escape needed (group_size=16 shards 8-way cleanly).
     match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 78.01 },
+    accuracy: { gsm8k_pct: 78.01, aime25_pct: 57.92 },
   },
   {
     // REAL — NVFP4 + DFlash (matched nvfp4-calibrated draft), tp8, trtllm_mha. Accept-len 4.04.
     match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 77.79 },
+    accuracy: { gsm8k_pct: 77.79, aime25_pct: 60.21 },
   },
   {
     // REAL — INT4 dense (mixed 4/8-bit MoE), tp8+ep8 (plain tp8 impossible: Marlin gs=128 'scales is not contiguous', same signature as H200).
     match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 67.25 },
+    accuracy: { gsm8k_pct: 67.25, aime25_pct: 63.54 },
   },
   {
     // REAL — INT4 + DFlash (matched int4-calibrated draft), tp8+ep8, trtllm_mha. Accept-len 4.01.
     match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)",
-    accuracy: { gsm8k_pct: 66.72 },
+    accuracy: { gsm8k_pct: 66.72, aime25_pct: 62.92 },
   },
 
   // ===== GB300 (4-GPU single node, tp 4) — ✅ REAL, full GSM8K =====
@@ -181,55 +210,55 @@ export const benchmarks = [
     match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 75.66 },
+    accuracy: { gsm8k_pct: 75.66, aime25_pct: 62.50 },
   },
   {
     // ✅ REAL — 4×GB300, BF16 + DFlash (matched bf16 draft), tp4, trtllm_mha. Accept-len 4.17.
     match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 76.19 },
+    accuracy: { gsm8k_pct: 76.19, aime25_pct: 65.83 },
   },
   {
     // ✅ REAL — 4×GB300, FP8 dense, tp4, backend auto→trtllm_mha.
     match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 71.87 },
+    accuracy: { gsm8k_pct: 71.87, aime25_pct: 63.12 },
   },
   {
     // ✅ REAL — 4×GB300, FP8 + DFlash (matched fp8-calibrated draft), tp4, trtllm_mha. Accept-len 4.05.
     match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 72.02 },
+    accuracy: { gsm8k_pct: 72.02, aime25_pct: 63.12 },
   },
   {
     // ✅ REAL — 4×GB300, NVFP4 dense, tp4, backend auto→trtllm_mha.
     match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 78.39 },
+    accuracy: { gsm8k_pct: 78.39, aime25_pct: 60.00 },
   },
   {
     // ✅ REAL — 4×GB300, NVFP4 + DFlash (matched nvfp4-calibrated draft), tp4, trtllm_mha. Accept-len 4.02.
     match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 74.53 },
+    accuracy: { gsm8k_pct: 74.53, aime25_pct: 60.00 },
   },
   {
     // ✅ REAL — 4×GB300, INT4 dense (mixed 4/8-bit MoE, needs #29761), tp4, backend auto→trtllm_mha.
     match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 66.79 },
+    accuracy: { gsm8k_pct: 66.79, aime25_pct: 64.79 },
   },
   {
     // ✅ REAL — 4×GB300, INT4 + DFlash (matched int4-calibrated draft), tp4, trtllm_mha. Accept-len 3.80.
     match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" },
     verified: true,
     sglang_version: "PR #29446 + #29761 (both merged to main)",
-    accuracy: { gsm8k_pct: 67.02 },
+    accuracy: { gsm8k_pct: 67.02, aime25_pct: 61.04 },
   },
 ];
diff --git a/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx b/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx
index 8cf0109f7..4f88be2b5 100644
--- a/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx
+++ b/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx
@@ -115,6 +115,18 @@ export const config = {
 `# pip install git+https://github.com/sgl-project/sgl-eval
 sgl-eval run gsm8k \\
   --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
+  --num-threads 128`,
+      // sgl-eval's --thinking sets the generic 'thinking' key, which Laguna's template
+      // ignores (it gates on enable_thinking — see Configuration Tips: Thinking). To run
+      // AIME25 with thinking, serve with a copy of the model's chat template whose
+      // enable_thinking default is flipped to true, passed via --chat-template.
+      aime25_pct:
+`# pip install git+https://github.com/sgl-project/sgl-eval
+# Serve with an enable_thinking=true chat template (see Configuration Tips: Thinking).
+sgl-eval run aime25 \\
+  --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
+  --n-repeats 16 --max-tokens 64000 \\
+  --temperature 1.0 --top-p 0.95 --thinking \\
   --num-threads 128`,
     },
     numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 },
@@ -122,11 +134,12 @@ sgl-eval run gsm8k \\
 
   // No variant-wide accuracy default; real numbers are per-cell in laguna-xs21-benchmarks.jsx.
   defaultAccuracy: {
-    default: { gsm8k_pct: null },
+    default: { gsm8k_pct: null, aime25_pct: null },
   },
 
   accuracyLabels: [
     ["gsm8k_pct", "GSM8K", "%"],
+    ["aime25_pct", "AIME25", "%"],
   ],
 
   // Dedicated image built for this cookbook page (PR #29446 + #29761 preinstalled on cu13).