From 476c946543971a36bf98e979f50889d94832b768 Mon Sep 17 00:00:00 2001 From: Jimmy Shong <69131491+Jiminator@users.noreply.github.com> Date: Thu, 2 Jul 2026 05:05:33 -0700 Subject: [PATCH] [Doc] Cookbook: Laguna-XS-2.1 (DFlash low-latency + high-throughput) (#29884) --- .../autoregressive/Poolside/Laguna-XS-2.1.mdx | 241 +++++++ docs_new/cookbook/autoregressive/intro.mdx | 2 +- docs_new/docs.json | 1 + .../poolside/laguna-xs21-benchmarks.jsx | 235 +++++++ .../snippets/configs/poolside/laguna-xs21.jsx | 587 ++++++++++++++++++ 5 files changed, 1065 insertions(+), 1 deletion(-) create mode 100644 docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx create mode 100644 docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx create mode 100644 docs_new/src/snippets/configs/poolside/laguna-xs21.jsx diff --git a/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx b/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx new file mode 100644 index 000000000..39e6cf1b1 --- /dev/null +++ b/docs_new/cookbook/autoregressive/Poolside/Laguna-XS-2.1.mdx @@ -0,0 +1,241 @@ +--- +title: Laguna-XS-2.1 +description: "Deploy poolside's Laguna-XS-2.1 — a 33B hybrid-SWA Mixture-of-Experts model (3B active) for agentic coding — with SGLang on NVIDIA H200, B300, and GB300 in BF16, FP8, NVFP4, and INT4." +tag: NEW +--- + +## Deployment + + + + + +Laguna-XS-2.1 support is fully merged to SGLang `main` ([PR #29446](https://github.com/sgl-project/sglang/pull/29446): DFlash speculative decoding + shared-expert fix; [PR #29761](https://github.com/sgl-project/sglang/pull/29761): INT4 loader fix). Any build at or past their merge covers every cell below. + +The model ships custom config code on the Hub, so `--trust-remote-code` is required (included in the launch commands). + + + + + +```bash Command +pip install -U uv +uv venv --python 3.12 && source .venv/bin/activate + +git clone https://github.com/sgl-project/sglang.git +cd sglang +uv pip install -e python +``` + +Then run the **Python** output of the command panel below in that environment. + + + + + +```bash Command +docker pull lmsysorg/sglang:dev-cu13-laguna-xs-2-1 +``` + +For how to launch the image, see [Install → Method 3: Using Docker](../../../docs/get-started/install#method-3-using-docker). Substitute the inner `sglang serve ...` with what the command generator below produces. + + + + + + + +Pick your hardware + quantization + strategy to generate the launch command. The two serving strategies cover the common operating points: + +- **Low-latency** — DFlash speculative decoding with a matched draft model. Pick for chat and interactive agents. +- **High-throughput** — plain serving. Best for batch workloads, where speculation's draft + rejection overhead costs more than it saves. + +On the 8-GPU HGX platforms (H200 / B300), BF16 and NVFP4 run plain `--tp 8`; FP8 and INT4 run `--tp 8 --ep-size 8` because their quantization scales cannot shard the MoE 8-way (see [Configuration Tips](#2-configuration-tips)). The 4-GPU GB300 node runs plain `--tp 4` throughout. + +import { Deployment } from "/src/snippets/_deployment.jsx"; +import { config } from "/src/snippets/configs/poolside/laguna-xs21.jsx"; +import { benchmarks } from "/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx"; + + + +## Playground + +The Playground is where you experiment with **SGLang features beyond the verified matrix**. The Deploy panel above only emits combinations that have been signed off; the Playground lets you turn on additional knobs (TP degree, parsers) on top of whichever cell the Deploy panel is currently showing. + +import { Playground } from "/src/snippets/_playground.jsx"; + + + +## 1. Model Introduction + +[Laguna-XS-2.1](https://huggingface.co/poolside/Laguna-XS-2.1) is an open-weight **33B-parameter** hybrid sliding-window-attention MoE model (**~3B active per token**) from [poolside](https://poolside.ai), built for agentic coding and long-horizon software engineering — the extra-small sibling of [Laguna-M.1](./Laguna-M.1). + +**Key Features:** + +- **Sparse MoE**: 40 layers, 256 routed experts, top-8 routing. +- **Hybrid attention**: 30 sliding-window layers (window 512) interleaved with 10 full-attention layers; 48 Q / 8 KV heads. +- **Long context**: 262,144 tokens (RoPE + YaRN on the full-attention layers). +- **DFlash drafts**: matched draft models (5-layer, ~0.9 GB) ship per quantization for low-latency serving. +- **Hybrid reasoning**: `…` toggled per request via `chat_template_kwargs={"enable_thinking": …}`. + +**Available quantizations:** + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PrecisionTarget modelDraft model
BF16[`poolside/Laguna-XS-2.1`](https://huggingface.co/poolside/Laguna-XS-2.1)[`poolside/Laguna-XS-2.1-DFlash`](https://huggingface.co/poolside/Laguna-XS-2.1-DFlash)
FP8[`poolside/Laguna-XS-2.1-FP8`](https://huggingface.co/poolside/Laguna-XS-2.1-FP8)[`poolside/Laguna-XS-2.1-DFlash-FP8`](https://huggingface.co/poolside/Laguna-XS-2.1-DFlash-FP8)
NVFP4[`poolside/Laguna-XS-2.1-NVFP4`](https://huggingface.co/poolside/Laguna-XS-2.1-NVFP4)[`poolside/Laguna-XS-2.1-DFlash-NVFP4`](https://huggingface.co/poolside/Laguna-XS-2.1-DFlash-NVFP4)
INT4[`poolside/Laguna-XS-2.1-INT4`](https://huggingface.co/poolside/Laguna-XS-2.1-INT4)[`poolside/Laguna-XS-2.1-DFlash-INT4`](https://huggingface.co/poolside/Laguna-XS-2.1-DFlash-INT4)
+ +The drafts themselves are small bf16 models, each *calibrated against its quantized target* — always pair a target with its matched draft (mixing precisions degrades accept-length). + +**License:** Apache 2.0 + +**Resources:** [Hugging Face](https://huggingface.co/poolside/Laguna-XS-2.1) · [Release blog post](https://poolside.ai/blog/laguna-a-deeper-dive) · [API platform](https://platform.poolside.ai). + +## 2. Configuration Tips + +**Attention backend** + +Leave `--attention-backend` unset for High-throughput cells — auto-select is correct (`fa3` on Hopper, `trtllm_mha` on Blackwell). With DFlash active, auto-select instead falls back to `flashinfer`, which breaks this hybrid-SWA model at `tp ≥ 4` on Blackwell (greedy GSM8K 76% → 28%), so the Low-latency commands pin the target backend explicitly. Leave `--speculative-draft-attention-backend` unset. Never use `triton` attention with Laguna (GSM8K 13%). + +**Quantized checkpoints cap plain TP at 4** + +`moe_intermediate_size=512` with FP8 block `[128,128]` / INT4 `group_size=128` scales cannot shard 8-way (512/8 = 64 < 128 granularity): FP8 fails at weight creation, INT4 crashes in the Marlin kernel, on any hardware. The generated 8-GPU FP8/INT4 commands therefore use `--tp 8 --ep-size 8` — expert parallelism keeps whole experts per rank, using all 8 GPUs on one instance. FP8 additionally needs `SGLANG_SHARED_EXPERT_TP1=1` (its shared expert is also block-quantized; INT4's stays bf16). Alternatives: plain `--tp 4`, or `--tp 4 --dp-size 2`. Accuracy is parallelism-independent within eval noise (verified tp1 ≡ tp4 on GB300 and tp4 ≡ tp8+ep8 on H200). + +**DFlash memory** + +Low-latency cells carry `--mem-fraction-static 0.7`: the default fraction OOMs in the draft vocab all-gather at `tp 4` on GB300. Dense cells use the default heuristic. + +**INT4 is mixed-precision** + +The INT4 checkpoint quantizes MoE layers in mixed 4-bit / 8-bit config groups. Builds older than [PR #29761](https://github.com/sgl-project/sglang/pull/29761) crash at load with `KeyError: 'Linear'`. + +**Chat template** + +On transformers ≥ 5.10 the standalone `chat_template.jinja` auto-loads — no flag needed (the server logs `Auto-detected template features: reasoning_parser=poolside_v1, ...`). On older transformers (≤ ~5.8) the `{% include %}` stub in `tokenizer_config.json` cannot resolve and the server silently falls back to a generic template — pass `--chat-template /chat_template.jinja` explicitly there. + +**Thinking** + +Off by default; opt in per request with `extra_body={"chat_template_kwargs": {"enable_thinking": True}}`. The template gates on `enable_thinking` — the generic `thinking` key is ignored. + +**Served model id** + +The server registers the model under whatever you pass to `--model-path`; a client's `model` field must match it (`poolside/Laguna-XS-2.1`, or the `-FP8` / `-NVFP4` / `-INT4` id). + +## 3. Advanced Usage + +### 3.1 DFlash Speculative Decoding + +DFlash is a block-wise speculative decoder: the 5-layer draft proposes a block of tokens and the target verifies the whole block in one forward pass, so only target-approved tokens are emitted — output quality is the target's by construction (GSM8K matches dense within noise on every quantization). The speedup lever is **accept-length**, the number of draft tokens surviving verification per target step: + +- Measured ~6 tokens/step at `tp 1`, ~4 at `tp 4` (greedy GSM8K, matched-precision pairs; ~3 under mixed reasoning-heavy traffic; FP8 reached 6.75 at `tp 8 + ep 8` on H200) — versus 1 token/step dense. +- Best for interactive / few-stream serving. Under batch-saturated load prefer High-throughput: once the GPU is compute-bound, draft + rejected-token overhead costs aggregate throughput. +- The generated commands always pair the draft calibrated for the selected target precision. + +### 3.2 Reasoning + +Launch with `--reasoning-parser poolside_v1` (baked into every generated command). Reasoning is opt-in via `enable_thinking=True`; the `` trace lands in `message.reasoning_content`, separate from the final answer in `message.content`. + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +response = client.chat.completions.create( + model="poolside/Laguna-XS-2.1", + messages=[{"role": "user", "content": "What is 15% of 240? Explain briefly."}], + max_tokens=2048, + extra_body={"chat_template_kwargs": {"enable_thinking": True}}, +) + +message = response.choices[0].message +print("=============== Reasoning ===============") +print(message.reasoning_content) +print("=============== Answer ==================") +print(message.content) +``` + + + + +XS-2.1 is an extra-small model — give it generous `max_tokens` when thinking is enabled +(hard problems regularly reason for thousands of tokens), and keep thinking off for +short-form tasks. + + +### 3.3 Tool Calling + +Launch with `--tool-call-parser poolside_v1` (baked into every generated command). The parser converts Laguna's `` output into the standard OpenAI `tool_calls` structure. Tool calling works with reasoning off (the default). + + + +```python Example +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY") + +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "The city name"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } +] + +response = client.chat.completions.create( + model="poolside/Laguna-XS-2.1", + messages=[{"role": "user", "content": "What's the weather in Beijing?"}], + tools=tools, +) + +message = response.choices[0].message +if message.tool_calls: + for call in message.tool_calls: + print(f"Tool: {call.function.name}") + print(f"Args: {call.function.arguments}") +``` + + diff --git a/docs_new/cookbook/autoregressive/intro.mdx b/docs_new/cookbook/autoregressive/intro.mdx index 4cf117fca..e6b6a97c0 100644 --- a/docs_new/cookbook/autoregressive/intro.mdx +++ b/docs_new/cookbook/autoregressive/intro.mdx @@ -142,7 +142,7 @@ metatags: diff --git a/docs_new/docs.json b/docs_new/docs.json index 24285957e..136b250ef 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -1150,6 +1150,7 @@ { "group": "Poolside", "pages": [ + "cookbook/autoregressive/Poolside/Laguna-XS-2.1", "cookbook/autoregressive/Poolside/Laguna-M.1", "cookbook/autoregressive/Poolside/Laguna-XS.2" ] diff --git a/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx b/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx new file mode 100644 index 000000000..c4b9e56be --- /dev/null +++ b/docs_new/src/snippets/configs/poolside/laguna-xs21-benchmarks.jsx @@ -0,0 +1,235 @@ +// Laguna-XS-2.1 benchmarks — one entry per cell `match` (same 5 keys as laguna-xs21.jsx cells). +// +// All numbers below are REAL measured values; cells without measurements are bare `{ match }` +// pending stubs (the card renders "pending"). NO fabricated/dummy numbers. +// +// REAL GSM8K (sgl-eval `run gsm8k`, FULL 1319 questions, greedy/non-thinking, chat template +// auto-loaded), measured on a 4×GB300 single node at tp 4: +// +// high-throughput (dense, backend auto→trtllm_mha): +// BF16 75.66% · FP8 71.87% · NVFP4 78.39% · INT4 66.79% +// low-latency (DFlash, --attention-backend trtllm_mha, matched-precision draft): +// BF16 76.19% (accept-len 4.17) · FP8 72.02% (4.05) · NVFP4 74.53% (4.02) · INT4 67.02% (3.80) +// +// Spec == dense within noise on every quant → DFlash is accuracy-neutral, as expected for +// verification-based speculation. Accept-length is the speedup lever (~4× fewer target steps +// at tp=4; ~5.7–6.8 accept-len measured at tp=1 on the same pairs). +// +// Backend caveats baked into the configs (do not "simplify" them away): +// - DFlash cells pin --attention-backend trtllm_mha on Blackwell: with speculation active, +// auto-select falls back to flashinfer, which breaks this hybrid-SWA model at tp≥4 +// (GSM8K 28% vs 76%, reproduced + single-variable-bisected on GB300). +// - `triton` attention is broken for Laguna (13.2% GSM8K) — never use it here. +// - Known open question: FP8/INT4 score ~5/~7 pts higher under flashinfer at tp≤2 than under +// trtllm_mha/fa4 (which agree with each other); bf16/nvfp4 are backend-invariant. Ground +// truth (HF eager reference) not yet established — the trtllm_mha numbers are shipped since +// that is the only tp≥4-viable backend. +// +// REAL GSM8K on an 8×H200 HGX node (sgl-eval `run gsm8k`, FULL 1319 questions, greedy/ +// non-thinking), backend fa3 (= the Hopper auto-select for dense; pinned for DFlash): +// +// high-throughput (dense): BF16 76.12% (tp 8) · FP8 73.54% (tp8+ep8) · INT4 67.02% (tp8+ep8) +// low-latency (DFlash, fa3): BF16 75.97% (tp 8) · FP8 74.53% (tp8+ep8) · INT4 66.57% (tp8+ep8) +// accept-lengths (matched-precision draft, greedy GSM8K): BF16 ~3.9 (bs=1) · FP8 6.75 · INT4 ~5. +// +// Spec == dense within noise on every quant, same as GB300 — DFlash is accuracy-neutral. +// INT4+DFlash's first full-set EP8 run drew 64.52% (2pt below the tp4 reference at 66.41%); +// a same-command repeat scored 66.57%, back in the reference cluster — the two EP8 draws +// alone span 2.05pt, comparable to the ~1pt spread FP8-dense showed across its own three +// independent full-set measurements (74.53 / 74.30 / 73.54). Confirmed ordinary eval noise, +// not an EP8/DFlash/INT4 interaction; 66.57% (the reproducing value) is shipped here. +// +// FP8/INT4 run --tp 8 --ep-size 8 (NOT plain tp 8, which fails at weight load — see +// laguna-xs21.jsx header comment for why: moe_intermediate_size=512 with FP8 block +// [128,128] / INT4 gs=128 scales can't shard 8-way). EP keeps whole experts per rank, +// sidestepping the shard-granularity wall entirely, so both quantizations use all 8 GPUs +// on one instance. FP8 additionally needs SGLANG_SHARED_EXPERT_TP1=1 (its shared expert +// is also block-quantized; INT4's stays bf16, no flag needed). The checks that make plain +// tp 8 fail are pure shard arithmetic with no arch branch → any 8-way plain-TP fails the +// same way, hence the B300 fp8/int4 cells also carry tp8+ep8. +// +// sglang_version = PR #29446 (DFlash + SGLANG_SHARED_EXPERT_TP1 fix) + PR #29761 (INT4 +// mixed-precision MoE load fix) — BOTH MERGED to main as of 2026-07-02. +// +// REAL GSM8K for the B300 column (sgl-eval `run gsm8k`, FULL 1319 questions, greedy/ +// non-thinking): the B300 cells' exact command shapes were run at tp8 as 2x(4xGB300) +// over MNNVL (NCCL_MNNVL_ENABLE/NCCL_CUMEM_ENABLE/MC_FORCE_MNNVL) — GB300 and B300 are +// the same Blackwell-Ultra 288GB GPU and the shard math (tp8; ep8 for fp8/int4) is +// identical to a single 8-GPU B300 node, so the accuracy measurement carries. Perf +// numbers (TTFT/throughput) were NOT taken from that topology and are left pending. +// +// high-throughput (dense): BF16 75.59% (tp8) | FP8 71.19% (tp8+ep8+flag) | +// NVFP4 78.01% (tp8) | INT4 67.25% (tp8+ep8) +// low-latency (DFlash, trtllm_mha): BF16 75.36% (4.08) | FP8 71.87% (4.05) | +// NVFP4 77.79% (4.04) | INT4 66.72% (4.01) +// Every cell at parity with its tp4-GB300 and H200 references; NVFP4 needs NO escape +// (group_size=16 divides the 64-wide tp8 shard — unlike FP8 [128,128] / INT4 gs=128). + +export const benchmarks = [ + // ===== H200 (8-GPU HGX; bf16 tp 8, fp8/int4 tp8+ep8) — ✅ REAL, full GSM8K ===== + { + // ✅ REAL — 8×H200, BF16 dense, tp8, backend fa3 (Hopper auto-select). + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 76.12 }, + }, + { + // ✅ REAL — 8×H200, BF16 + DFlash (matched bf16 draft), tp8, fa3. Accept-len 3.05 + // (mixed eval traffic; ~3.9 greedy GSM8K bs=1). + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 75.97 }, + }, + { + // ✅ REAL — 8×H200, FP8 dense, tp8+ep8+SGLANG_SHARED_EXPERT_TP1=1 (plain tp8 impossible: + // block-FP8 scale granularity), fa3. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 73.54 }, + }, + { + // ✅ REAL — 8×H200, FP8 + DFlash (matched fp8-calibrated draft), tp8+ep8+flag, fa3. + // Accept-len 6.75. + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 74.53 }, + }, + { + // ✅ REAL — 8×H200, INT4 dense (mixed 4/8-bit MoE, needs #29761), tp8+ep8 (plain tp8 + // impossible: Marlin gs=128 scale layout; no shared-expert flag needed), fa3. + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 67.02 }, + }, + { + // ✅ REAL — 8×H200, INT4 + DFlash (matched int4-calibrated draft), tp8+ep8, fa3. + // Accept-len ~5. First run drew 64.52%, repeat scored this value (66.57%) — confirmed + // ordinary eval noise, not a real EP8/DFlash interaction; see header note. + match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 66.57 }, + }, + + // ===== B300 (8-GPU HGX; bf16/nvfp4 tp 8, fp8/int4 tp8+ep8) — REAL, full GSM8K ===== + // (accuracy measured as 2x(4xGB300) tp8/MNNVL — same GPU + shard math as one B300 node) + { + // REAL — BF16 dense, tp8, backend auto->trtllm_mha. + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 75.59 }, + }, + { + // REAL — BF16 + DFlash (matched bf16 draft), tp8, trtllm_mha. Accept-len 4.08. + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 75.36 }, + }, + { + // REAL — FP8 dense, tp8+ep8+SGLANG_SHARED_EXPERT_TP1=1 (plain tp8 impossible: block-FP8 scale granularity). + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 71.19 }, + }, + { + // REAL — FP8 + DFlash (matched fp8-calibrated draft), tp8+ep8+flag, trtllm_mha. Accept-len 4.05. + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 71.87 }, + }, + { + // REAL — NVFP4 dense, tp8 — NO escape needed (group_size=16 shards 8-way cleanly). + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 78.01 }, + }, + { + // REAL — NVFP4 + DFlash (matched nvfp4-calibrated draft), tp8, trtllm_mha. Accept-len 4.04. + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 77.79 }, + }, + { + // REAL — INT4 dense (mixed 4/8-bit MoE), tp8+ep8 (plain tp8 impossible: Marlin gs=128 'scales is not contiguous', same signature as H200). + match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 67.25 }, + }, + { + // REAL — INT4 + DFlash (matched int4-calibrated draft), tp8+ep8, trtllm_mha. Accept-len 4.01. + match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main; run @ main 0543246184)", + accuracy: { gsm8k_pct: 66.72 }, + }, + + // ===== GB300 (4-GPU single node, tp 4) — ✅ REAL, full GSM8K ===== + { + // ✅ REAL — 4×GB300, BF16 dense, tp4, backend auto→trtllm_mha. + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 75.66 }, + }, + { + // ✅ REAL — 4×GB300, BF16 + DFlash (matched bf16 draft), tp4, trtllm_mha. Accept-len 4.17. + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 76.19 }, + }, + { + // ✅ REAL — 4×GB300, FP8 dense, tp4, backend auto→trtllm_mha. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 71.87 }, + }, + { + // ✅ REAL — 4×GB300, FP8 + DFlash (matched fp8-calibrated draft), tp4, trtllm_mha. Accept-len 4.05. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 72.02 }, + }, + { + // ✅ REAL — 4×GB300, NVFP4 dense, tp4, backend auto→trtllm_mha. + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 78.39 }, + }, + { + // ✅ REAL — 4×GB300, NVFP4 + DFlash (matched nvfp4-calibrated draft), tp4, trtllm_mha. Accept-len 4.02. + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 74.53 }, + }, + { + // ✅ REAL — 4×GB300, INT4 dense (mixed 4/8-bit MoE, needs #29761), tp4, backend auto→trtllm_mha. + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 66.79 }, + }, + { + // ✅ REAL — 4×GB300, INT4 + DFlash (matched int4-calibrated draft), tp4, trtllm_mha. Accept-len 3.80. + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + sglang_version: "PR #29446 + #29761 (both merged to main)", + accuracy: { gsm8k_pct: 67.02 }, + }, +]; diff --git a/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx b/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx new file mode 100644 index 000000000..8cf0109f7 --- /dev/null +++ b/docs_new/src/snippets/configs/poolside/laguna-xs21.jsx @@ -0,0 +1,587 @@ +// Laguna-XS-2.1 (poolside) — config-driven cookbook page. +// Consumed by the shared _deployment.jsx + _playground.jsx engines (no model code there). +// +// Build: the `laguna` model type (hybrid SWA + MoE) is on SGLang main. Two extra pieces, +// BOTH MERGED to main as of 2026-07-02 — no branch/cherry-pick needed: +// - INT4: poolside/Laguna-XS-2.1-INT4 is a MIXED-precision compressed-tensors MoE +// (4-bit + 8-bit config groups, regex targets, no "Linear" group) — needs PR #29761 +// or it crashes at load with KeyError: 'Linear'. +// - Low-Latency (DFlash speculative decoding) + the 8-GPU FP8 recipe below both need +// PR #29446 (Laguna XS-2.1 DFlash support + SGLANG_SHARED_EXPERT_TP1 shared-expert fix). +// +// Attention backend (IMPORTANT — Laguna is hybrid-SWA and backend-sensitive): +// - Dense (High-Throughput): leave --attention-backend UNSET. Auto-select is correct: +// fa3 on Hopper (H200), trtllm_mha on Blackwell (B300/GB300). +// - DFlash (Low-Latency): auto-select is NOT safe — with a speculative algorithm active +// the resolver falls back to flashinfer, which on Blackwell HALVES greedy GSM8K at +// tp=4 (76.2% -> 28%, reproduced+bisected on GB300). Every LL cell therefore PINS the +// target backend explicitly: fa3 on H200, trtllm_mha on Blackwell. The draft worker +// cannot run trtllm_mha and auto-falls-back to flashinfer — measured identical to a +// forced fa4 draft (82.5% vs 81.5% holdout, accept-len 4.63 both), so it is left auto. +// - NEVER use --attention-backend triton for Laguna: 13.2% GSM8K (broken SWA handling) +// plus a CUBLAS crash at tp=4 CUDA-graph capture. +// +// Draft/target precision ALWAYS matches: each quantized target pairs with the DFlash draft +// calibrated for it (…-DFlash, …-DFlash-FP8, …-DFlash-NVFP4, …-DFlash-INT4). The drafts +// themselves are small bf16 5-layer models (~0.9 GB) — the suffix is the calibration target. +// +// Memory: DFlash cells carry --mem-fraction-static 0.7 — at tp=4 on GB300 the default +// fraction OOMs in the draft vocab all-gather ("Failed to CUDA calloc"); 0.7 is validated. +// Dense cells use the default heuristic (validated at defaults on GB300). +// +// TP/EP on the 8-GPU HGX platforms (H200/B300): plain --tp 8 works for BF16, but the +// quantized checkpoints cap PLAIN TP at 4 — moe_intermediate_size=512 with FP8 block +// [128,128] / INT4 group_size=128 scales cannot shard 8-way (512/8 = 64 < 128 granularity +// → FP8 ValueError at weight create, INT4 Marlin scale-contiguity crash; reproduced on +// 8×H200, and the checks are pure shard arithmetic — arch-independent, so this is not an +// H200-only limitation). To still use all 8 GPUs on a single instance, FP8/INT4 cells use +// `--tp 8 --ep-size 8` instead: EP keeps whole experts per rank (256 experts ÷ 8 = 32, +// avoiding the 512-dim MoE intermediate shard entirely) which fixes the *routed* experts +// for both precisions. FP8's shared expert is ALSO block-quantized (unlike INT4's, which +// stays bf16), so FP8 additionally needs `SGLANG_SHARED_EXPERT_TP1=1` (replicates the +// shared expert instead of TP-sharding it — see PR #29446). GB300 (4-GPU node) uses plain +// `--tp 4` throughout since 4 GPUs is already inside the plain-TP ceiling. +// +// NVFP4 is Blackwell-only → no h200×nvfp4 cells (same rule as Laguna-M.1). +// +// verified:true = ran that command shape and it served correctly + passed full GSM8K +// (see laguna-xs21-benchmarks.jsx). GB300 cells verified (4×GB300, tp 4); H200 cells +// verified (8×H200: bf16 tp8, fp8/int4 tp8+ep8). B300 cells verified with the identical +// commands run as tp8 across 2×(4×GB300) over MNNVL — same GPU (GB300/B300 = Blackwell +// Ultra, 288GB), same shard math, so the accuracy measurement carries; single-node B300 +// re-timing (perf) is the only thing not covered by that setup. + +export const config = { + modelName: "Laguna-XS-2.1", + + supportedHardware: ["h200", "b300", "gb300"], + + variants: [ + { id: "default", label: "Default" }, + ], + + quantizations: [ + { id: "bf16", label: "BF16" }, + { id: "fp8", label: "FP8" }, + { id: "nvfp4", label: "NVFP4" }, + { id: "int4", label: "INT4" }, + ], + + // Two operating points: + // low-latency = DFlash speculative decoding (matched-precision draft) — interactive / + // few-stream serving; measured accept-length ~3.8–4.2 at tp=4 (~5.7–6.8 at tp=1). + // high-throughput = plain serving (no speculation) — batch-saturated workloads, where + // speculation's draft+rejection overhead costs more than it saves. + strategies: [ + { id: "low-latency", label: "Low-latency" }, + { id: "high-throughput", label: "High-throughput" }, + ], + + nodesOptions: [ + { id: "single", label: "Single Node" }, + ], + + modelNames: { + "default|bf16": "poolside/Laguna-XS-2.1", + "default|fp8": "poolside/Laguna-XS-2.1-FP8", + "default|nvfp4": "poolside/Laguna-XS-2.1-NVFP4", + "default|int4": "poolside/Laguna-XS-2.1-INT4", + }, + + placeholders: { + HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" }, + PORT: { target: "command", label: "Bind port", default: "30000" }, + HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "" }, + CURL_HOST: { target: "curl", label: "Server host", default: "localhost" }, + CURL_PORT: { target: "curl", label: "Server port", default: "30000" }, + }, + + curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\ +-H 'Content-Type: application/json' \\ +-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"Hello"}] }'`, + + benchmarkCommands: { + speed: +`python3 -m sglang.bench_serving \\ + --backend sglang \\ + --host {{CURL_HOST}} --port {{CURL_PORT}} \\ + --model {{MODEL_NAME}} \\ + --dataset-name {{DATASET}} \\ + --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}}`, + // GSM8K is the required accuracy sanity on every verified cell (cookbook_guide §3), via sgl-eval. + accuracy: { + gsm8k_pct: +`# pip install git+https://github.com/sgl-project/sgl-eval +sgl-eval run gsm8k \\ + --base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\ + --num-threads 128`, + }, + numPromptsByConc: { 1: 8, 16: 32, 64: 128, 128: 256, 256: 512, 1024: 2048, 4096: 4096 }, + }, + + // No variant-wide accuracy default; real numbers are per-cell in laguna-xs21-benchmarks.jsx. + defaultAccuracy: { + default: { gsm8k_pct: null }, + }, + + accuracyLabels: [ + ["gsm8k_pct", "GSM8K", "%"], + ], + + // Dedicated image built for this cookbook page (PR #29446 + #29761 preinstalled on cu13). + dockerImages: { + h200: "lmsysorg/sglang:dev-cu13-laguna-xs-2-1", + b300: "lmsysorg/sglang:dev-cu13-laguna-xs-2-1", + gb300: "lmsysorg/sglang:dev-cu13-laguna-xs-2-1", + }, + + github: { + cookbookModel: "poolside/Laguna-XS-2.1", + }, + + playgroundFeatures: { + + // Hybrid-SWA GQA model (48 Q / 8 KV heads) — TP shards cleanly at 1/2/4/8. + // Accuracy verified TP-independent on the trtllm_mha backend (tp1 == tp4 on GB300). + // No DP-Attention / CP knobs: unvalidated on this model family — not exposed. + attention: { + knobs: [ + { id: "tp", label: "TP", values: [null, 1, 2, 4, 8] }, + ], + }, + + // Reasoning + tool-call parsers (poolside_v1, same family as Laguna-M.1 / XS.2). + // ALSO baked into every Deploy cell below. The chat template auto-detects both + // (`Auto-detected template features: reasoning_parser=poolside_v1, tool_call_parser=poolside_v1`), + // so these are explicit-but-redundant on transformers ≥ 5.10. + parsers: { + items: [ + { id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser poolside_v1" }, + { id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser poolside_v1" }, + ], + }, + }, + + // Cells: (h200 × {bf16,fp8,int4} + b300/gb300 × {bf16,fp8,nvfp4,int4}) × {low-latency, high-throughput}. + // Draft model precision always matches the target's. + cells: [ + + // ══════════════ NVIDIA Hopper H200 (8-GPU HGX) — BF16 / FP8 / INT4 — VERIFIED ══════════════ + // All 6 cells ran on 8×H200 with full-GSM8K accuracy (laguna-xs21-benchmarks.jsx). + // Dense auto-selects fa3 on Hopper (no flag). LL pins fa3 (DFlash-safe on Hopper; + // with a spec algorithm active, auto would fall back to flashinfer). + // FP8/INT4 use --tp 8 --ep-size 8 to use all 8 GPUs on one instance (plain --tp 8 + // crashes at weight load for both — see header comment). FP8 additionally needs + // SGLANG_SHARED_EXPERT_TP1=1 (its shared expert is block-quantized too). + { + // VERIFIED 8×H200 tp8: GSM8K 76.12% (full 1319, greedy). + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 8×H200 tp8: GSM8K 75.97%, accept-length 3.05 (matched bf16 draft; + // ~3.9 on greedy GSM8K at bs=1). + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend fa3", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 8×H200 tp8+ep8: GSM8K 73.54% (full 1319, greedy). + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 8×H200 tp8+ep8: GSM8K 74.53%, accept-length 6.75 (matched fp8-calibrated draft). + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--attention-backend fa3", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 8×H200 tp8+ep8: GSM8K 67.02% (full 1319, greedy). Mixed 4/8-bit MoE — + // needs a build ≥ PR #29761 (merged). No SGLANG_SHARED_EXPERT_TP1 needed — INT4's + // shared expert stays bf16 (its ignore-list keeps it unquantized), so it TP-shards + // freely under EP; only FP8's shared expert needs replication. + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 8×H200 tp8+ep8: GSM8K 66.57% (matched int4-calibrated draft), accept-length + // ~5. First run drew 64.52% — 2pt below the tp4 sibling (66.41%); a same-command repeat + // scored 66.57%, confirming ordinary eval noise (not an EP8/DFlash interaction). + match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--attention-backend fa3", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-INT4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ══════════════ NVIDIA Blackwell Ultra B300 (8-GPU HGX) — BF16 / FP8 / NVFP4 / INT4 ══════════════ + // Dense auto-selects trtllm_mha on Blackwell (no flag). LL MUST pin trtllm_mha — + // with DFlash active, auto falls back to flashinfer, which is broken for this + // hybrid-SWA model at tp≥4 (GSM8K 28% vs 76%; reproduced + bisected on GB300). + // VERIFIED: these exact command shapes ran as tp8 across 2×(4×GB300)/MNNVL — identical + // silicon + shard math to one 8-GPU B300 node — with full-GSM8K accuracy per cell + // (dense 75.59/71.19/78.01/67.25, DFlash 75.36/71.87/77.79/66.72 for bf16/fp8/nvfp4/int4). + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // Plain tp 8 fails at weight load (quantized MoE TP cap, arch-independent — see + // header comment); tp 8 + ep 8 uses all 8 GPUs instead (verified on 8×H200, same + // merged fix — pending measurement on this hardware). + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: ["SGLANG_SHARED_EXPERT_TP1=1"], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-NVFP4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // INT4 (mixed 4/8-bit compressed-tensors MoE) — needs a build ≥ PR #29761 (merged). + // tp 8 + ep 8 uses all 8 GPUs (verified on 8×H200 — pending measurement on this + // hardware); no SGLANG_SHARED_EXPERT_TP1 needed, INT4's shared expert stays bf16. + match: { hw: "b300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 8", + "--ep-size 8", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-INT4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + + // ══════════════ NVIDIA Grace-Blackwell GB300 (4-GPU single node) — VERIFIED ══════════════ + // All 8 cells ran on 4×GB300 (tp 4) with full-GSM8K accuracy (laguna-xs21-benchmarks.jsx): + // dense via backend auto-select (resolves trtllm_mha), DFlash with trtllm_mha pinned. + { + // VERIFIED 4×GB300 tp4: GSM8K 75.66% (full 1319, greedy). + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 76.19%, accept-length 4.17 (matched bf16 draft). + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 71.87%. + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 72.02%, accept-length 4.05 (matched fp8-calibrated draft). + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-FP8", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 78.39%. + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 74.53%, accept-length 4.02 (matched nvfp4-calibrated draft). + match: { hw: "gb300", variant: "default", quant: "nvfp4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-NVFP4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 66.79%. Mixed 4/8-bit MoE — needs a build ≥ PR #29761 (merged). + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + // VERIFIED 4×GB300 tp4: GSM8K 67.02%, accept-length 3.80 (matched int4-calibrated draft). + match: { hw: "gb300", variant: "default", quant: "int4", strategy: "low-latency", nodes: "single" }, + verified: true, + env: [], + flags: [ + "--model-path {{MODEL_NAME}}", + "--trust-remote-code", + "--reasoning-parser poolside_v1", + "--tool-call-parser poolside_v1", + "--tp 4", + "--attention-backend trtllm_mha", + "--speculative-algorithm DFLASH", + "--speculative-draft-model-path poolside/Laguna-XS-2.1-DFlash-INT4", + "--page-size 1", + "--mem-fraction-static 0.7", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + ], +};