[Cookbook] Laguna-M.1: enable FP8 on Blackwell + drop provisional AIME numbers (#28664)

2026-06-18 09:23:53 -07:00
parent 0eded9e208
commit f7632ef860
3 changed files with 119 additions and 64 deletions
@@ -110,7 +110,7 @@ import { Playground } from "/src/snippets/_playground.jsx";
 ## 2. Configuration Tips

 - **Long-context memory**: M.1 is global-attention (no sliding-window), so the 262,144-token KV cache is large. If you hit OOM at full context, lower `--mem-fraction-static` or cap `--context-length`.
- **FP8 first launch**: FP8 triggers a multi-session DeepGEMM JIT pre-compile on first launch; pre-warm with `python3 -m sglang.compile_deep_gemm --model poolside/Laguna-M.1-FP8` to avoid paying it on every restart.
+- **FP8**: On **Blackwell** the recipe adds `--fp8-gemm-backend triton` — the compressed-tensors block-FP8 weight scales aren't UE8M0-packed, so the default DeepGEMM path emits garbage on Blackwell (sm_100); the Triton backend is correct (~19% slower). Temporary workaround pending [PR #28662](https://github.com/sgl-project/sglang/pull/28662) (which fixes the scales and restores the DeepGEMM fast path). On **Hopper (H200)** FP8 uses DeepGEMM with no extra flag — pre-warm its multi-session JIT with `python3 -m sglang.compile_deep_gemm --model poolside/Laguna-M.1-FP8` to avoid paying it on each restart.
 - **Parsers** (`poolside_v1`): for agentic / tool-using deployments enable the **Reasoning Parser** and **Tool Call Parser** in the Playground above — they emit `--reasoning-parser poolside_v1` (thinking → `reasoning_content`) and `--tool-call-parser poolside_v1` (structured `tool_calls`).
 - **Thinking default**: thinking is **off by default**; opt in per request with `extra_body={"chat_template_kwargs": {"enable_thinking": True}}`.
 - **Served model id**: the server registers the model under whatever you pass to `--model-path`, so a client's `model` field must match it — `poolside/Laguna-M.1` (BF16) or `poolside/Laguna-M.1-FP8` / `-NVFP4` for the quantized cells. The §3 examples use the BF16 id; swap in the id you launched.
@@ -1,24 +1,19 @@
 // Laguna-M.1 benchmarks — one entry per cell `match` (same 5 keys as laguna-m1.jsx cells).
 //
 // All numbers below are REAL measured values; cells without measurements are bare `{ match }`
-// pending stubs (the card renders "pending"). NO fabricated/dummy numbers remain.
-// (cookbook_guide §3 forbids fabricated numbers in a published page.)
+// pending stubs (the card renders "pending"). NO fabricated/dummy numbers.
+// Accuracy axis is GSM8K-only for now (AIME 25 will be re-added once truncation-free numbers exist).
 //
-// FP8 is HOPPER-ONLY (not compatible with Blackwell) → the only FP8 entry is H200; there are no
-// Blackwell FP8 entries. Blackwell cells are BF16 / NVFP4.
-//
-// REAL numbers (sgl-eval; GSM8K non-thinking; AIME via the enable_thinking wrapper, max_tokens=32768):
-//   H200 BF16 (tp8): GSM8K 93.02% · AIME25 53.33% overall (~0.80 stop-only) · perf (cc1, cc128).
-//   H200 FP8  (tp8): GSM8K 93.25% · AIME25 50.0% overall (~0.79 stop-only). g_proj FP8 fix validated; no perf (BF16-only scope).
-//   B200 BF16 (tp8): GSM8K 91.88% · AIME25 66.88% (n_repeats=16) · perf A/B (cc1, cc128).
-//   B200 NVFP4 (tp8): GSM8K 89.38%.
-// ⚠️ AIME OVERALL is depressed by ~33–37% truncation at the 32k cap (M.1 reasoning is long → no boxed
-// answer → scored 0); stop-only (~0.80) is the truer signal, and a 48–64k cap would lift the overall.
+// REAL GSM8K (sgl-eval `run gsm8k`, full 1319, non-thinking):
+//   H200 BF16 (tp8): 93.02%  · perf bench_serving random 4096/1024 (cc1, cc128).
+//   H200 FP8  (tp8): 93.25%.
+//   B200 BF16 (tp8): 91.88%  · perf A/B (cc1, cc128).
+//   B200 FP8  (tp8): 93.78%  — with `--fp8-gemm-backend triton` (DeepGEMM UE8M0 workaround; ~19% slower).
+//   B200 NVFP4 (tp8): 89.38%.
 // (perf tokens_per_sec_per_gpu = measured output tok/s ÷ 8 GPUs; TTFT = median.)
 //
-// sglang_version reflects the REQUIRED build = PR #28400 (per-element gating) + PR #28604
-// (global-attention SWA fix). The plain #28400 wheel (0.5.14.dev20260618+g343aeeef39) is NOT
-// enough — it crashes M.1 under load. H200 numbers were taken on a main build @ 3f668733.
+// sglang_version = the build the numbers ran on (PR #28400 + #28604, +#28649 for FP8 load,
+// +#28662/triton-workaround for Blackwell FP8). H200 numbers taken on a main build @ 3f668733.

 export const benchmarks = [
  // ===== H200 — BF16 / FP8 =====
@@ -31,25 +26,23 @@ export const benchmarks = [
      // cc=1: median TTFT 81.89 ms, median TPOT 8.91 ms, output 109.96 tok/s (÷8 ≈ 13.7/GPU).
      { workload: { dataset: "random", isl: 4096, osl: 1024, max_concurrency: 1 },
        ttft_ms: 81.9, tpot_ms: 8.91, tokens_per_sec_per_gpu: 13.7 },
-      // cc=128: median TTFT 200.11 ms (mean 1221), median TPOT 52.09 ms, output 2266 tok/s (÷8 ≈ 283/GPU); total 11311 tok/s.
+      // cc=128: median TTFT 200.11 ms (mean 1221), median TPOT 52.09 ms, output 2266 tok/s (÷8 ≈ 283/GPU).
      { workload: { dataset: "random", isl: 4096, osl: 1024, max_concurrency: 128 },
        ttft_ms: 200.1, tpot_ms: 52.1, tokens_per_sec_per_gpu: 283 },
    ],
-    accuracy: { gsm8k_pct: 93.02, aime25_pct: 53.33 }, // AIME overall, n_repeats=1 (32k-truncation-limited, ~33% trunc; stop-only ~0.80)
+    accuracy: { gsm8k_pct: 93.02 },
  },
  {
-    // ✅ REAL — 8xH200, FP8, tp8. GSM8K 93.25 + AIME25 0.50 (overall). The g_proj quant fix is
-    // validated (FP8 now loads past layer 0). Perf not measured on FP8 (BF16-only scope) → no speed row.
-    // FP8 is Hopper-only — not compatible with Blackwell (no Blackwell FP8 entry below).
+    // ✅ REAL — 8xH200, FP8, tp8. GSM8K 93.25%. (Hopper: no --fp8-gemm-backend flag needed.)
    match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
    verified: true,
-    sglang_version: "main @ 3f668733 (#28400 + #28604 + g_proj FP8 fix)",
-    accuracy: { gsm8k_pct: 93.25, aime25_pct: 50.0 }, // AIME overall (32k-truncation-limited; stop-only ~0.79)
+    sglang_version: "main @ 3f668733 (#28400 + #28604 + g_proj FP8 fix #28649)",
+    accuracy: { gsm8k_pct: 93.25 },
  },

-  // ===== B200 (8-GPU HGX) — BF16 / NVFP4 =====
+  // ===== B200 (8-GPU HGX) — BF16 / FP8 / NVFP4 =====
  {
-    // ✅ REAL — 8xB200, BF16, tp8. GSM8K 91.88 + AIME25 66.88; perf A/B (laguna-m1-results.md).
+    // ✅ REAL — 8xB200, BF16, tp8. GSM8K 91.88; perf A/B (laguna-m1-results.md).
    match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
    verified: true,
    sglang_version: "PR #28400 + #28604",
@@ -59,23 +52,33 @@ export const benchmarks = [
      { workload: { dataset: "random", isl: 4096, osl: 1024, max_concurrency: 128 },
        ttft_ms: 170, tpot_ms: 43.3, tokens_per_sec_per_gpu: 331 },
    ],
-    accuracy: { gsm8k_pct: 91.88, aime25_pct: 66.88 },
+    accuracy: { gsm8k_pct: 91.88 },
  },
  {
-    // ✅ REAL (GSM8K only) — 8xB200, NVFP4, tp8. GSM8K 89.38 (laguna-m1-results.md).
-    // AIME + perf not measured yet → omitted (card shows them pending).
+    // ✅ REAL — 8xB200, FP8, tp8, with --fp8-gemm-backend triton. GSM8K 93.78% (full 1319,
+    // laguna-m1-results.md). Matches H200 FP8 (93.25) within noise; sits above B200 NVFP4 (89.38).
+    match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+    verified: true,
+    sglang_version: "main + #28649 + --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround; fix = PR #28662)",
+    accuracy: { gsm8k_pct: 93.78 },
+  },
+  {
+    // ✅ REAL — 8xB200, NVFP4, tp8. GSM8K 89.38% (laguna-m1-results.md).
    match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
    verified: true,
    sglang_version: "PR #28400 + #28604",
    accuracy: { gsm8k_pct: 89.38 },
  },

-  // ===== B300 / GB200 / GB300 — BF16 / NVFP4, UNVERIFIED, no data yet → bare "pending" stubs
-  // (no fabricated numbers; FP8 is Hopper-only so no Blackwell FP8 entries). =====
+  // ===== B300 / GB200 / GB300 — BF16 / FP8 / NVFP4, UNVERIFIED → bare "pending" stubs (no
+  // fabricated numbers). Blackwell FP8 cells carry --fp8-gemm-backend triton in the config. =====
  { match: { hw: "b300",  variant: "default", quant: "bf16",  strategy: "balanced", nodes: "single" } },
+  { match: { hw: "b300",  variant: "default", quant: "fp8",   strategy: "balanced", nodes: "single" } },
  { match: { hw: "b300",  variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } },
  { match: { hw: "gb200", variant: "default", quant: "bf16",  strategy: "balanced", nodes: "single" } },
+  { match: { hw: "gb200", variant: "default", quant: "fp8",   strategy: "balanced", nodes: "single" } },
  { match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } },
  { match: { hw: "gb300", variant: "default", quant: "bf16",  strategy: "balanced", nodes: "single" } },
+  { match: { hw: "gb300", variant: "default", quant: "fp8",   strategy: "balanced", nodes: "single" } },
  { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } },
 ];
@@ -4,7 +4,7 @@
 // Build: M.1 needs SGLang PR #28400 (softplus per-element output gating, MERGED) AND PR #28604
 // (global-attention SWA fix — M.1 is sliding_window=0 / all-global; without it M.1 crashes ~1s
 // into any concurrent batch with AssertionError: ... swa_lock_ref=0). Both are merged on main
-// (verified on a 3f668733 build). The shipped recipe carries NO workaround flag, but the pinned
+// (verified on a 3f668733 build). The shipped recipe carries NO SWA workaround flag, but the pinned
 // build MUST contain BOTH; the #28400-merge wheel 0.5.14.dev20260618+g343aeeef39 is #28400-ONLY
 // and crashes under load. Pin dockerImages + benchmarks.sglang_version to a build at a commit
 // ≥ #28604. See /sgl-workspace/laguna-m1-day0-checklist.md (step 2) + laguna-m1-results.md.
@@ -13,9 +13,12 @@
 //
 // Hardware: H200 (Hopper) + B200/B300/GB200/GB300 (Blackwell).
 //   - BF16 runs everywhere.
-//   - FP8 is HOPPER-ONLY — not compatible with Blackwell (use NVFP4 for low-precision there).
+//   - FP8 runs everywhere. On Blackwell (sm_100) the compressed-tensors block-FP8 weight scales
+//     aren't UE8M0-packed, so the default DeepGEMM path produces garbage → the Blackwell FP8 cells
+//     add `--fp8-gemm-backend triton` (correct, ~19% slower than DeepGEMM). Temporary until the
+//     ue8m0-requant fix (PR #28662) lands; H200 FP8 (Hopper) is unaffected and needs no flag.
 //   - NVFP4 is Blackwell-only.
-// So the only quant×hw combos with a cell are: H200×{BF16,FP8} and each Blackwell×{BF16,NVFP4}.
+// Cells: H200×{BF16,FP8}; each Blackwell×{BF16,FP8,NVFP4}.
 // TP: 8-GPU HGX nodes (H200/B200/B300) → --tp 8 (the maintainer's baseline); GB200/GB300
 // (Grace-Blackwell, typically 4-GPU single node) → --tp 4. Adjust --tp to your node size.
 //
@@ -74,39 +77,32 @@ export const config = {
  --dataset-name {{DATASET}} \\
  --random-input-len {{ISL}} --random-output-len {{OSL}} \\
  --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`,
-    // GSM8K sanity is the REQUIRED minimum on every verified cell (cookbook_guide §3); AIME 25
-    // (thinking ON) is the harder accuracy check (model_support_guide). All via sgl-eval.
-    // NOTE: M.1 needs enable_thinking, not sgl-eval's --thinking key (which the template ignores)
-    // — thinking evals were run via the enable_thinking wrapper (laguna-m1-results.md).
+    // GSM8K is the required accuracy sanity on every verified cell (cookbook_guide §3), via sgl-eval.
+    // (AIME 25 to be added back once truncation-free numbers are measured.)
    accuracy: {
      gsm8k_pct:
 `# pip install git+https://github.com/sgl-project/sgl-eval
 sgl-eval run gsm8k \\
  --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
  --num-threads 128`,
-      aime25_pct:
-`# pip install git+https://github.com/sgl-project/sgl-eval
-sgl-eval run aime25 --thinking \\
-  --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
-  --num-threads 32`,
    },
    numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 },
  },

-  // Accuracy here varies by QUANT (BF16 vs NVFP4), so real numbers live per-cell in
-  // laguna-m1-benchmarks.jsx (which overrides this). Keep null = no variant-wide default.
+  // Hardware-independent accuracy default (null = no variant-wide default; real numbers are per-cell
+  // in laguna-m1-benchmarks.jsx).
  defaultAccuracy: {
-    default: { gsm8k_pct: null, aime25_pct: null },
+    default: { gsm8k_pct: null },
  },

  accuracyLabels: [
-    ["gsm8k_pct",  "GSM8K",   "%"],
-    ["aime25_pct", "AIME 25", "%"],
+    ["gsm8k_pct", "GSM8K", "%"],
  ],

  // Pinned nightly with the Laguna-M.1 build (PR #28400 + #28604 + #28649; cu13 covers H200 + all Blackwell).
-  // dev-cu13-618-nightly was generated after the FP8 g_proj fix (#28649) landed, so it serves H200 FP8 as
-  // well as BF16/NVFP4. (Equivalent pip nightly: 0.5.14.dev20260618+g97e3b8998d.)
+  // dev-cu13-618-nightly was generated after the FP8 g_proj fix (#28649) landed, so it serves FP8 too.
+  // (Equivalent pip nightly: 0.5.14.dev20260618+g97e3b8998d.) Blackwell FP8 additionally needs the
+  // --fp8-gemm-backend triton flag (in those cells) until PR #28662 merges.
  dockerImages: {
    h200:  "lmsysorg/sglang:dev-cu13-618-nightly",
    b200:  "lmsysorg/sglang:dev-cu13-618-nightly",
@@ -153,9 +149,9 @@ sgl-eval run aime25 --thinking \\
    },
  },

-  // One Balanced cell per valid (hw × quant): H200×{BF16,FP8}, each Blackwell×{BF16,NVFP4}.
-  // FP8 is Hopper-only (no Blackwell FP8 cell — those combos grey out). Baseline recipe
-  // (parsers poolside_v1, NO --trust-remote-code) baked into every cell.
+  // One Balanced cell per valid (hw × quant): H200×{BF16,FP8}; each Blackwell×{BF16,FP8,NVFP4}.
+  // Blackwell FP8 cells add `--fp8-gemm-backend triton` (DeepGEMM UE8M0 workaround, pending #28662);
+  // H200 FP8 needs no such flag. Baseline recipe (parsers poolside_v1, NO --trust-remote-code) on every cell.
  // TP: H200/B200/B300 = --tp 8; GB200/GB300 = --tp 4 (4-GPU single node).
  // verified:true = ran that exact command on that hardware and it served correctly + passed a
  // GSM8K-class eval. Absent verified = yellow/unverified badge.
@@ -176,10 +172,9 @@ sgl-eval run aime25 --thinking \\
      ],
    },
    {
-      // VERIFIED on 8xH200 (FP8, tp8): GSM8K 93.25 + AIME25 0.50; g_proj FP8 quant fix validated.
-      // FP8 needs the g_proj fix (PR #28649, MERGED) on top of #28400+#28604 — the pinned
-      // dev-cu13-618-nightly image includes all three.
-      // FP8 is Hopper-only — it is NOT compatible with Blackwell, so there is no Blackwell FP8 cell.
+      // VERIFIED on 8xH200 (FP8, tp8): GSM8K 93.25%. FP8 needs the g_proj fix (PR #28649, MERGED) on
+      // top of #28400+#28604 — the pinned dev-cu13-618-nightly image has it. Hopper does NOT hit the
+      // Blackwell DeepGEMM UE8M0 issue, so no --fp8-gemm-backend flag here.
      match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
      verified: true,
      env: [],
@@ -192,10 +187,9 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
-    // ===== NVIDIA Blackwell B200 (8-GPU HGX) — BF16 / NVFP4 (FP8 is Hopper-only) =====
+    // ===== NVIDIA Blackwell B200 (8-GPU HGX) — BF16 / FP8 / NVFP4 =====
    {
-      // VERIFIED on 8xB200 (BF16, tp8): served clean under batched shared-prefix load,
-      // GSM8K 91.88% + AIME25 66.88% (laguna-m1-results.md).
+      // VERIFIED on 8xB200 (BF16, tp8): served clean under batched shared-prefix load, GSM8K 91.88%.
      match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
      verified: true,
      env: [],
@@ -209,8 +203,24 @@ sgl-eval run aime25 --thinking \\
      ],
    },
    {
-      // VERIFIED on 8xB200 (NVFP4, tp8): GSM8K 89.38% (laguna-m1-results.md). tp8 now matches
-      // the shipped recipe.
+      // VERIFIED on 8xB200 (FP8, tp8): GSM8K 93.78% with --fp8-gemm-backend triton (laguna-m1-results.md).
+      // The triton backend sidesteps the DeepGEMM UE8M0 weight-scale bug on Blackwell (~19% slower than
+      // the DeepGEMM fast path). Drop the flag once PR #28662 (ue8m0 requant) merges.
+      match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+      verified: true,
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--reasoning-parser poolside_v1",
+        "--tool-call-parser poolside_v1",
+        "--tp 8",
+        "--fp8-gemm-backend triton",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      // VERIFIED on 8xB200 (NVFP4, tp8): GSM8K 89.38% (laguna-m1-results.md).
      match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
      verified: true,
      env: [],
@@ -223,7 +233,7 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
-    // ===== NVIDIA Blackwell B300 (8-GPU HGX) — BF16 / NVFP4 (UNVERIFIED; FP8 is Hopper-only) =====
+    // ===== NVIDIA Blackwell B300 (8-GPU HGX) — BF16 / FP8 / NVFP4 (UNVERIFIED) =====
    {
      match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
      env: [],
@@ -236,6 +246,20 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
+    {
+      // FP8 on Blackwell → --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround, pending #28662).
+      match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--reasoning-parser poolside_v1",
+        "--tool-call-parser poolside_v1",
+        "--tp 8",
+        "--fp8-gemm-backend triton",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
    {
      match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
      env: [],
@@ -248,7 +272,7 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
-    // ===== NVIDIA Grace-Blackwell GB200 (4-GPU single node) — BF16 / NVFP4 (UNVERIFIED; FP8 Hopper-only) =====
+    // ===== NVIDIA Grace-Blackwell GB200 (4-GPU single node) — BF16 / FP8 / NVFP4 (UNVERIFIED) =====
    {
      match: { hw: "gb200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
      env: [],
@@ -261,6 +285,20 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
+    {
+      // FP8 on Blackwell → --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround, pending #28662).
+      match: { hw: "gb200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--reasoning-parser poolside_v1",
+        "--tool-call-parser poolside_v1",
+        "--tp 4",
+        "--fp8-gemm-backend triton",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
    {
      match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
      env: [],
@@ -273,7 +311,7 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
-    // ===== NVIDIA Grace-Blackwell GB300 (4-GPU single node) — BF16 / NVFP4 (UNVERIFIED; FP8 Hopper-only) =====
+    // ===== NVIDIA Grace-Blackwell GB300 (4-GPU single node) — BF16 / FP8 / NVFP4 (UNVERIFIED) =====
    {
      match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" },
      env: [],
@@ -286,6 +324,20 @@ sgl-eval run aime25 --thinking \\
        "--port {{PORT}}",
      ],
    },
+    {
+      // FP8 on Blackwell → --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround, pending #28662).
+      match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" },
+      env: [],
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--reasoning-parser poolside_v1",
+        "--tool-call-parser poolside_v1",
+        "--tp 4",
+        "--fp8-gemm-backend triton",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
    {
      match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" },
      env: [],