From f7632ef8603bf966cc2062cd45e6fd369ec83931 Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:23:53 -0700 Subject: [PATCH] [Cookbook] Laguna-M.1: enable FP8 on Blackwell + drop provisional AIME numbers (#28664) --- .../autoregressive/Poolside/Laguna-M.1.mdx | 2 +- .../configs/poolside/laguna-m1-benchmarks.jsx | 61 ++++----- .../snippets/configs/poolside/laguna-m1.jsx | 120 +++++++++++++----- 3 files changed, 119 insertions(+), 64 deletions(-) diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx index cac3e707c..10459ddc1 100644 --- a/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-M.1.mdx @@ -110,7 +110,7 @@ import { Playground } from "/src/snippets/_playground.jsx"; ## 2. Configuration Tips - **Long-context memory**: M.1 is global-attention (no sliding-window), so the 262,144-token KV cache is large. If you hit OOM at full context, lower `--mem-fraction-static` or cap `--context-length`. -- **FP8 first launch**: FP8 triggers a multi-session DeepGEMM JIT pre-compile on first launch; pre-warm with `python3 -m sglang.compile_deep_gemm --model poolside/Laguna-M.1-FP8` to avoid paying it on every restart. +- **FP8**: On **Blackwell** the recipe adds `--fp8-gemm-backend triton` — the compressed-tensors block-FP8 weight scales aren't UE8M0-packed, so the default DeepGEMM path emits garbage on Blackwell (sm_100); the Triton backend is correct (~19% slower). Temporary workaround pending [PR #28662](https://github.com/sgl-project/sglang/pull/28662) (which fixes the scales and restores the DeepGEMM fast path). On **Hopper (H200)** FP8 uses DeepGEMM with no extra flag — pre-warm its multi-session JIT with `python3 -m sglang.compile_deep_gemm --model poolside/Laguna-M.1-FP8` to avoid paying it on each restart. - **Parsers** (`poolside_v1`): for agentic / tool-using deployments enable the **Reasoning Parser** and **Tool Call Parser** in the Playground above — they emit `--reasoning-parser poolside_v1` (thinking → `reasoning_content`) and `--tool-call-parser poolside_v1` (structured `tool_calls`). - **Thinking default**: thinking is **off by default**; opt in per request with `extra_body={"chat_template_kwargs": {"enable_thinking": True}}`. - **Served model id**: the server registers the model under whatever you pass to `--model-path`, so a client's `model` field must match it — `poolside/Laguna-M.1` (BF16) or `poolside/Laguna-M.1-FP8` / `-NVFP4` for the quantized cells. The §3 examples use the BF16 id; swap in the id you launched. diff --git a/docs_new/src/snippets/configs/poolside/laguna-m1-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-m1-benchmarks.jsx index e03d63142..4b8058127 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-m1-benchmarks.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-m1-benchmarks.jsx @@ -1,24 +1,19 @@ // Laguna-M.1 benchmarks — one entry per cell `match` (same 5 keys as laguna-m1.jsx cells). // // All numbers below are REAL measured values; cells without measurements are bare `{ match }` -// pending stubs (the card renders "pending"). NO fabricated/dummy numbers remain. -// (cookbook_guide §3 forbids fabricated numbers in a published page.) +// pending stubs (the card renders "pending"). NO fabricated/dummy numbers. +// Accuracy axis is GSM8K-only for now (AIME 25 will be re-added once truncation-free numbers exist). // -// FP8 is HOPPER-ONLY (not compatible with Blackwell) → the only FP8 entry is H200; there are no -// Blackwell FP8 entries. Blackwell cells are BF16 / NVFP4. -// -// REAL numbers (sgl-eval; GSM8K non-thinking; AIME via the enable_thinking wrapper, max_tokens=32768): -// H200 BF16 (tp8): GSM8K 93.02% · AIME25 53.33% overall (~0.80 stop-only) · perf (cc1, cc128). -// H200 FP8 (tp8): GSM8K 93.25% · AIME25 50.0% overall (~0.79 stop-only). g_proj FP8 fix validated; no perf (BF16-only scope). -// B200 BF16 (tp8): GSM8K 91.88% · AIME25 66.88% (n_repeats=16) · perf A/B (cc1, cc128). -// B200 NVFP4 (tp8): GSM8K 89.38%. -// ⚠️ AIME OVERALL is depressed by ~33–37% truncation at the 32k cap (M.1 reasoning is long → no boxed -// answer → scored 0); stop-only (~0.80) is the truer signal, and a 48–64k cap would lift the overall. +// REAL GSM8K (sgl-eval `run gsm8k`, full 1319, non-thinking): +// H200 BF16 (tp8): 93.02% · perf bench_serving random 4096/1024 (cc1, cc128). +// H200 FP8 (tp8): 93.25%. +// B200 BF16 (tp8): 91.88% · perf A/B (cc1, cc128). +// B200 FP8 (tp8): 93.78% — with `--fp8-gemm-backend triton` (DeepGEMM UE8M0 workaround; ~19% slower). +// B200 NVFP4 (tp8): 89.38%. // (perf tokens_per_sec_per_gpu = measured output tok/s ÷ 8 GPUs; TTFT = median.) // -// sglang_version reflects the REQUIRED build = PR #28400 (per-element gating) + PR #28604 -// (global-attention SWA fix). The plain #28400 wheel (0.5.14.dev20260618+g343aeeef39) is NOT -// enough — it crashes M.1 under load. H200 numbers were taken on a main build @ 3f668733. +// sglang_version = the build the numbers ran on (PR #28400 + #28604, +#28649 for FP8 load, +// +#28662/triton-workaround for Blackwell FP8). H200 numbers taken on a main build @ 3f668733. export const benchmarks = [ // ===== H200 — BF16 / FP8 ===== @@ -31,25 +26,23 @@ export const benchmarks = [ // cc=1: median TTFT 81.89 ms, median TPOT 8.91 ms, output 109.96 tok/s (÷8 ≈ 13.7/GPU). { workload: { dataset: "random", isl: 4096, osl: 1024, max_concurrency: 1 }, ttft_ms: 81.9, tpot_ms: 8.91, tokens_per_sec_per_gpu: 13.7 }, - // cc=128: median TTFT 200.11 ms (mean 1221), median TPOT 52.09 ms, output 2266 tok/s (÷8 ≈ 283/GPU); total 11311 tok/s. + // cc=128: median TTFT 200.11 ms (mean 1221), median TPOT 52.09 ms, output 2266 tok/s (÷8 ≈ 283/GPU). { workload: { dataset: "random", isl: 4096, osl: 1024, max_concurrency: 128 }, ttft_ms: 200.1, tpot_ms: 52.1, tokens_per_sec_per_gpu: 283 }, ], - accuracy: { gsm8k_pct: 93.02, aime25_pct: 53.33 }, // AIME overall, n_repeats=1 (32k-truncation-limited, ~33% trunc; stop-only ~0.80) + accuracy: { gsm8k_pct: 93.02 }, }, { - // ✅ REAL — 8xH200, FP8, tp8. GSM8K 93.25 + AIME25 0.50 (overall). The g_proj quant fix is - // validated (FP8 now loads past layer 0). Perf not measured on FP8 (BF16-only scope) → no speed row. - // FP8 is Hopper-only — not compatible with Blackwell (no Blackwell FP8 entry below). + // ✅ REAL — 8xH200, FP8, tp8. GSM8K 93.25%. (Hopper: no --fp8-gemm-backend flag needed.) match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, verified: true, - sglang_version: "main @ 3f668733 (#28400 + #28604 + g_proj FP8 fix)", - accuracy: { gsm8k_pct: 93.25, aime25_pct: 50.0 }, // AIME overall (32k-truncation-limited; stop-only ~0.79) + sglang_version: "main @ 3f668733 (#28400 + #28604 + g_proj FP8 fix #28649)", + accuracy: { gsm8k_pct: 93.25 }, }, - // ===== B200 (8-GPU HGX) — BF16 / NVFP4 ===== + // ===== B200 (8-GPU HGX) — BF16 / FP8 / NVFP4 ===== { - // ✅ REAL — 8xB200, BF16, tp8. GSM8K 91.88 + AIME25 66.88; perf A/B (laguna-m1-results.md). + // ✅ REAL — 8xB200, BF16, tp8. GSM8K 91.88; perf A/B (laguna-m1-results.md). match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, sglang_version: "PR #28400 + #28604", @@ -59,23 +52,33 @@ export const benchmarks = [ { workload: { dataset: "random", isl: 4096, osl: 1024, max_concurrency: 128 }, ttft_ms: 170, tpot_ms: 43.3, tokens_per_sec_per_gpu: 331 }, ], - accuracy: { gsm8k_pct: 91.88, aime25_pct: 66.88 }, + accuracy: { gsm8k_pct: 91.88 }, }, { - // ✅ REAL (GSM8K only) — 8xB200, NVFP4, tp8. GSM8K 89.38 (laguna-m1-results.md). - // AIME + perf not measured yet → omitted (card shows them pending). + // ✅ REAL — 8xB200, FP8, tp8, with --fp8-gemm-backend triton. GSM8K 93.78% (full 1319, + // laguna-m1-results.md). Matches H200 FP8 (93.25) within noise; sits above B200 NVFP4 (89.38). + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + sglang_version: "main + #28649 + --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround; fix = PR #28662)", + accuracy: { gsm8k_pct: 93.78 }, + }, + { + // ✅ REAL — 8xB200, NVFP4, tp8. GSM8K 89.38% (laguna-m1-results.md). match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, verified: true, sglang_version: "PR #28400 + #28604", accuracy: { gsm8k_pct: 89.38 }, }, - // ===== B300 / GB200 / GB300 — BF16 / NVFP4, UNVERIFIED, no data yet → bare "pending" stubs - // (no fabricated numbers; FP8 is Hopper-only so no Blackwell FP8 entries). ===== + // ===== B300 / GB200 / GB300 — BF16 / FP8 / NVFP4, UNVERIFIED → bare "pending" stubs (no + // fabricated numbers). Blackwell FP8 cells carry --fp8-gemm-backend triton in the config. ===== { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } }, + { match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, { match: { hw: "gb200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } }, + { match: { hw: "gb200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } }, { match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" } }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" } }, ]; diff --git a/docs_new/src/snippets/configs/poolside/laguna-m1.jsx b/docs_new/src/snippets/configs/poolside/laguna-m1.jsx index bfb434171..35ad464e5 100644 --- a/docs_new/src/snippets/configs/poolside/laguna-m1.jsx +++ b/docs_new/src/snippets/configs/poolside/laguna-m1.jsx @@ -4,7 +4,7 @@ // Build: M.1 needs SGLang PR #28400 (softplus per-element output gating, MERGED) AND PR #28604 // (global-attention SWA fix — M.1 is sliding_window=0 / all-global; without it M.1 crashes ~1s // into any concurrent batch with AssertionError: ... swa_lock_ref=0). Both are merged on main -// (verified on a 3f668733 build). The shipped recipe carries NO workaround flag, but the pinned +// (verified on a 3f668733 build). The shipped recipe carries NO SWA workaround flag, but the pinned // build MUST contain BOTH; the #28400-merge wheel 0.5.14.dev20260618+g343aeeef39 is #28400-ONLY // and crashes under load. Pin dockerImages + benchmarks.sglang_version to a build at a commit // ≥ #28604. See /sgl-workspace/laguna-m1-day0-checklist.md (step 2) + laguna-m1-results.md. @@ -13,9 +13,12 @@ // // Hardware: H200 (Hopper) + B200/B300/GB200/GB300 (Blackwell). // - BF16 runs everywhere. -// - FP8 is HOPPER-ONLY — not compatible with Blackwell (use NVFP4 for low-precision there). +// - FP8 runs everywhere. On Blackwell (sm_100) the compressed-tensors block-FP8 weight scales +// aren't UE8M0-packed, so the default DeepGEMM path produces garbage → the Blackwell FP8 cells +// add `--fp8-gemm-backend triton` (correct, ~19% slower than DeepGEMM). Temporary until the +// ue8m0-requant fix (PR #28662) lands; H200 FP8 (Hopper) is unaffected and needs no flag. // - NVFP4 is Blackwell-only. -// So the only quant×hw combos with a cell are: H200×{BF16,FP8} and each Blackwell×{BF16,NVFP4}. +// Cells: H200×{BF16,FP8}; each Blackwell×{BF16,FP8,NVFP4}. // TP: 8-GPU HGX nodes (H200/B200/B300) → --tp 8 (the maintainer's baseline); GB200/GB300 // (Grace-Blackwell, typically 4-GPU single node) → --tp 4. Adjust --tp to your node size. // @@ -74,39 +77,32 @@ export const config = { --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} \\ --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, - // GSM8K sanity is the REQUIRED minimum on every verified cell (cookbook_guide §3); AIME 25 - // (thinking ON) is the harder accuracy check (model_support_guide). All via sgl-eval. - // NOTE: M.1 needs enable_thinking, not sgl-eval's --thinking key (which the template ignores) - // — thinking evals were run via the enable_thinking wrapper (laguna-m1-results.md). + // GSM8K is the required accuracy sanity on every verified cell (cookbook_guide §3), via sgl-eval. + // (AIME 25 to be added back once truncation-free numbers are measured.) accuracy: { gsm8k_pct: `# pip install git+https://github.com/sgl-project/sgl-eval sgl-eval run gsm8k \\ --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ --num-threads 128`, - aime25_pct: -`# pip install git+https://github.com/sgl-project/sgl-eval -sgl-eval run aime25 --thinking \\ - --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ - --num-threads 32`, }, numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 }, }, - // Accuracy here varies by QUANT (BF16 vs NVFP4), so real numbers live per-cell in - // laguna-m1-benchmarks.jsx (which overrides this). Keep null = no variant-wide default. + // Hardware-independent accuracy default (null = no variant-wide default; real numbers are per-cell + // in laguna-m1-benchmarks.jsx). defaultAccuracy: { - default: { gsm8k_pct: null, aime25_pct: null }, + default: { gsm8k_pct: null }, }, accuracyLabels: [ - ["gsm8k_pct", "GSM8K", "%"], - ["aime25_pct", "AIME 25", "%"], + ["gsm8k_pct", "GSM8K", "%"], ], // Pinned nightly with the Laguna-M.1 build (PR #28400 + #28604 + #28649; cu13 covers H200 + all Blackwell). - // dev-cu13-618-nightly was generated after the FP8 g_proj fix (#28649) landed, so it serves H200 FP8 as - // well as BF16/NVFP4. (Equivalent pip nightly: 0.5.14.dev20260618+g97e3b8998d.) + // dev-cu13-618-nightly was generated after the FP8 g_proj fix (#28649) landed, so it serves FP8 too. + // (Equivalent pip nightly: 0.5.14.dev20260618+g97e3b8998d.) Blackwell FP8 additionally needs the + // --fp8-gemm-backend triton flag (in those cells) until PR #28662 merges. dockerImages: { h200: "lmsysorg/sglang:dev-cu13-618-nightly", b200: "lmsysorg/sglang:dev-cu13-618-nightly", @@ -153,9 +149,9 @@ sgl-eval run aime25 --thinking \\ }, }, - // One Balanced cell per valid (hw × quant): H200×{BF16,FP8}, each Blackwell×{BF16,NVFP4}. - // FP8 is Hopper-only (no Blackwell FP8 cell — those combos grey out). Baseline recipe - // (parsers poolside_v1, NO --trust-remote-code) baked into every cell. + // One Balanced cell per valid (hw × quant): H200×{BF16,FP8}; each Blackwell×{BF16,FP8,NVFP4}. + // Blackwell FP8 cells add `--fp8-gemm-backend triton` (DeepGEMM UE8M0 workaround, pending #28662); + // H200 FP8 needs no such flag. Baseline recipe (parsers poolside_v1, NO --trust-remote-code) on every cell. // TP: H200/B200/B300 = --tp 8; GB200/GB300 = --tp 4 (4-GPU single node). // verified:true = ran that exact command on that hardware and it served correctly + passed a // GSM8K-class eval. Absent verified = yellow/unverified badge. @@ -176,10 +172,9 @@ sgl-eval run aime25 --thinking \\ ], }, { - // VERIFIED on 8xH200 (FP8, tp8): GSM8K 93.25 + AIME25 0.50; g_proj FP8 quant fix validated. - // FP8 needs the g_proj fix (PR #28649, MERGED) on top of #28400+#28604 — the pinned - // dev-cu13-618-nightly image includes all three. - // FP8 is Hopper-only — it is NOT compatible with Blackwell, so there is no Blackwell FP8 cell. + // VERIFIED on 8xH200 (FP8, tp8): GSM8K 93.25%. FP8 needs the g_proj fix (PR #28649, MERGED) on + // top of #28400+#28604 — the pinned dev-cu13-618-nightly image has it. Hopper does NOT hit the + // Blackwell DeepGEMM UE8M0 issue, so no --fp8-gemm-backend flag here. match: { hw: "h200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, verified: true, env: [], @@ -192,10 +187,9 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, - // ===== NVIDIA Blackwell B200 (8-GPU HGX) — BF16 / NVFP4 (FP8 is Hopper-only) ===== + // ===== NVIDIA Blackwell B200 (8-GPU HGX) — BF16 / FP8 / NVFP4 ===== { - // VERIFIED on 8xB200 (BF16, tp8): served clean under batched shared-prefix load, - // GSM8K 91.88% + AIME25 66.88% (laguna-m1-results.md). + // VERIFIED on 8xB200 (BF16, tp8): served clean under batched shared-prefix load, GSM8K 91.88%. match: { hw: "b200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, verified: true, env: [], @@ -209,8 +203,24 @@ sgl-eval run aime25 --thinking \\ ], }, { - // VERIFIED on 8xB200 (NVFP4, tp8): GSM8K 89.38% (laguna-m1-results.md). tp8 now matches - // the shipped recipe. + // VERIFIED on 8xB200 (FP8, tp8): GSM8K 93.78% with --fp8-gemm-backend triton (laguna-m1-results.md). + // The triton backend sidesteps the DeepGEMM UE8M0 weight-scale bug on Blackwell (~19% slower than + // the DeepGEMM fast path). Drop the flag once PR #28662 (ue8m0 requant) merges. + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--fp8-gemm-backend triton", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED on 8xB200 (NVFP4, tp8): GSM8K 89.38% (laguna-m1-results.md). match: { hw: "b200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, verified: true, env: [], @@ -223,7 +233,7 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, - // ===== NVIDIA Blackwell B300 (8-GPU HGX) — BF16 / NVFP4 (UNVERIFIED; FP8 is Hopper-only) ===== + // ===== NVIDIA Blackwell B300 (8-GPU HGX) — BF16 / FP8 / NVFP4 (UNVERIFIED) ===== { match: { hw: "b300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, env: [], @@ -236,6 +246,20 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, + { + // FP8 on Blackwell → --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround, pending #28662). + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--fp8-gemm-backend triton", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, { match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, env: [], @@ -248,7 +272,7 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, - // ===== NVIDIA Grace-Blackwell GB200 (4-GPU single node) — BF16 / NVFP4 (UNVERIFIED; FP8 Hopper-only) ===== + // ===== NVIDIA Grace-Blackwell GB200 (4-GPU single node) — BF16 / FP8 / NVFP4 (UNVERIFIED) ===== { match: { hw: "gb200", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, env: [], @@ -261,6 +285,20 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, + { + // FP8 on Blackwell → --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround, pending #28662). + match: { hw: "gb200", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--fp8-gemm-backend triton", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, { match: { hw: "gb200", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, env: [], @@ -273,7 +311,7 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, - // ===== NVIDIA Grace-Blackwell GB300 (4-GPU single node) — BF16 / NVFP4 (UNVERIFIED; FP8 Hopper-only) ===== + // ===== NVIDIA Grace-Blackwell GB300 (4-GPU single node) — BF16 / FP8 / NVFP4 (UNVERIFIED) ===== { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "balanced", nodes: "single" }, env: [], @@ -286,6 +324,20 @@ sgl-eval run aime25 --thinking \\ "--port {{PORT}}", ], }, + { + // FP8 on Blackwell → --fp8-gemm-backend triton (DeepGEMM UE8M0 workaround, pending #28662). + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "single" }, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--fp8-gemm-backend triton", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, { match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "balanced", nodes: "single" }, env: [],