diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx index c1076867b..d50fcf958 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx @@ -20,7 +20,7 @@ For how to launch the image, see [Install → Method 3: Using Docker](../../../d Pick your hardware + recipe to generate the launch command. Three serving strategies are covered: -- **Low-Latency** — fastest reply for a single user. Pick for chat. These recipes run NEXTN speculative decoding. +- **Low-Latency** — fastest reply for a single user. Pick for chat. These recipes run speculative decoding: pick **NEXTN** (built-in MTP layer, no extra checkpoint) or **DSPARK** (external draft model) in the **Spec Decode** selector of the Deploy panel. - **High-Throughput** — most tokens per second across many users. Best for batch jobs. These recipes turn speculative decoding off, since at saturation the draft/verify overhead outweighs the speedup. - **HiCache + Mooncake** — writes reusable prefixes to Mooncake L3 storage. Start the Mooncake services in §3.3 before launching the generated server command. @@ -50,6 +50,7 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports - **FP8** (blockwise E4M3): [inclusionAI/Ling-3.0-flash-fp8](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp8) - **INT4** (compressed-tensors W4A16): [inclusionAI/Ling-3.0-flash-int4](https://huggingface.co/inclusionAI/Ling-3.0-flash-int4) - **MXFP4**: [inclusionAI/Ling-3.0-flash-fp4](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp4) +- **DSPARK draft**: [inclusionAI/Ling-3.0-flash-dspark](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark) — 5-layer DSpark draft checkpoint for DSPARK speculative decoding **License:** MIT @@ -64,7 +65,8 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports - `--reasoning-parser ling3` and `--tool-call-parser ling3` enable Ling-3.0-specific reasoning and structured tool-call parsing; toggle them in the **Parsers** card of the [Playground](#playground). - Both the chat template and the `ling3` reasoning parser default to thinking on. A single request can turn it off with `"chat_template_kwargs": {"enable_thinking": false}` (see §3.1). - The BF16/FP8 recipes use `--mem-fraction-static 0.8`; INT4/MXFP4 use `0.85`. These values reserve the headroom used by the validated graph-enabled runs. -- The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes have it on; toggle it in the **Speculative Decoding** card of the [Playground](#playground). +- The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes default to it via the **Spec Decode** selector; toggle it in the **Speculative Decoding** card of the [Playground](#playground). +- DSPARK is the alternative speculative path: it drafts with the external [inclusionAI/Ling-3.0-flash-dspark](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark) checkpoint instead of the built-in MTP layer. Pick **DSPARK** in the Deploy panel's **Spec Decode** selector (also available in the [Playground](#playground)); it composes `--speculative-algorithm DSPARK`, the draft path, and `--enable-linear-replayssm-spec`, the KDA verify path that folds per-draft intermediate states onto a fixed ring instead of snapshotting full states. The draft block size auto-infers from the checkpoint; tune it with `--speculative-dspark-block-size`. The draft's block size of 8 makes the verify window 9 tokens, and the KDA ReplaySSM ring must be a power of two at least twice the window — the recipe therefore pins `--linear-replayssm-cache-len 32`, since the 16 default fails startup validation. Validated on BF16, 4×B200 TP4, thinking on: full GSM8K 96.66% with a 99.77% stop rate (NEXTN on the same setup: 96.44% / 99.62%); per-cell speed and accuracy numbers sit under the B200 low-latency DSPARK cell in the Deploy panel. - Native context is 256K; SGLang reads it from the checkpoint's `max_position_embeddings`, so no `--context-length` flag is needed. - The **HiCache** card in the [Playground](#playground) exposes the validated Mooncake L3 path. It adds the hybrid-KDA scheduler and prefix-key settings together; see §3.3 for the required services. diff --git a/docs/src/snippets/_playground.jsx b/docs/src/snippets/_playground.jsx index 5fe304d5e..2e07cd9f1 100644 --- a/docs/src/snippets/_playground.jsx +++ b/docs/src/snippets/_playground.jsx @@ -825,6 +825,8 @@ export const Playground = ({ config }) => { || head === "--speculative-eagle-topk" || head === "--speculative-num-draft-tokens" || head === "--speculative-dspark-block-size" + || head === "--enable-linear-replayssm-spec" + || head === "--linear-replayssm-cache-len" || head === "--speculative-ngram-max-bfs-breadth"; }); if (baseSpec.length === 0) return "off"; @@ -850,7 +852,8 @@ export const Playground = ({ config }) => { flags = h.stripFlagsByFirstToken(flags, [ "--speculative-algorithm", "--speculative-num-steps", "--speculative-eagle-topk", "--speculative-num-draft-tokens", - "--speculative-dspark-block-size", + "--speculative-dspark-block-size", "--enable-linear-replayssm-spec", + "--linear-replayssm-cache-len", "--speculative-ngram-max-bfs-breadth", ]); const preset = (fc.options || []).find((p) => p.id === value); diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx index e4d7a1e7a..3904b2612 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx @@ -12,14 +12,14 @@ export const benchmarks = [ // H200 + BF16 (TP4) // ==================================================================== { - match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, sglang_version: "PR #33561 @ c5071ded", accuracy: { gsm8k_pct: 96.59 }, }, { // Rejected by the full GSM8K gate at request 1319: a no-EOS runaway generated // >33k tokens. Recipe stays `verified: false` in ling-3.0-flash.jsx. - match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, accuracy: { gsm8k_pct: null }, notes: "Full GSM8K gate did not complete: one request ran away without emitting EOS (>33k generated tokens).", }, @@ -28,17 +28,17 @@ export const benchmarks = [ // H200 + FP8 (TP4 + EP4) // ==================================================================== { - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 95.83 }, }, { - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 96.51 }, }, { - match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e1a24a18", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, @@ -50,7 +50,7 @@ export const benchmarks = [ notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 32 shapes through batch 218.", }, { - match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e1a24a18", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, @@ -66,7 +66,7 @@ export const benchmarks = [ // H200 + HiCache (Mooncake tiered cache) // ==================================================================== { - match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", spec: "nextn", nodes: "single" }, sglang_version: "PR #33561 @ 51bcd89c", accuracy: { gsm8k_pct: 96.44 }, }, @@ -75,12 +75,30 @@ export const benchmarks = [ // B200 + BF16 (TP4) // ==================================================================== { - match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, - sglang_version: "PR #33561 @ c5071ded", - accuracy: { gsm8k_pct: 96.44 }, + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, + sglang_version: "PR #33561 @ 0e5e40d8f (dev-Ling-3.0-flash image)", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 171.57, tpot_ms: 2.34, tokens_per_sec_per_gpu: 886 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 248.30, tpot_ms: 5.29, tokens_per_sec_per_gpu: 5936 }, + ], + accuracy: { gsm8k_pct: 96.66 }, }, { - match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "dspark", nodes: "single" }, + sglang_version: "PR #33561 @ 0e5e40d8f (dev-Ling-3.0-flash image)", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 156.44, tpot_ms: 0.78, tokens_per_sec_per_gpu: 1902 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 247.27, tpot_ms: 2.87, tokens_per_sec_per_gpu: 8025 }, + ], + accuracy: { gsm8k_pct: 96.59 }, + notes: "Average accept length 6.22 (200q GSM8K window); deterministic 20-request stop rate 100%.", + }, + { + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ c5071ded", accuracy: { gsm8k_pct: 96.51 }, }, @@ -89,17 +107,17 @@ export const benchmarks = [ // B200 + FP8 (TP4 + EP4) // ==================================================================== { - match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 96.59 }, }, { - match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 97.04 }, }, { - match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e1a24a18", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, @@ -111,7 +129,7 @@ export const benchmarks = [ notes: "Full GSM8K stop rate 99.70%; default decode CUDA Graph captured 40 shapes through batch 305.", }, { - match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e1a24a18", speed: [ { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, @@ -129,14 +147,14 @@ export const benchmarks = [ // TODO: both cells are `verified: true` (gated on the final head) but the GSM8K // percentages were not recorded in the PR body or in the verifying commits — // fill from the run logs. - { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" } }, // ==================================================================== // GB300 + FP8 (TP4 + EP4) // ==================================================================== // TODO: both cells are `verified: true` on the final head; the 96.66% / 96.44% // pair in the PR body predates the TP+EP change — fill with the re-measured values. - { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" } }, - { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" } }, + { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" } }, ]; diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx index 7b9b329c8..8845b0b37 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx @@ -4,9 +4,7 @@ export const config = { supportedHardware: ["h20-3e", "h200", "h800", "h100", "b200", "gb300"], groupHardware: false, - variants: [ - { id: "default", label: "Ling-3.0-flash" }, - ], + variants: [{ id: "default", label: "Ling-3.0-flash" }], quantizations: [ { id: "bf16", label: "BF16" }, { id: "fp8", label: "FP8" }, @@ -18,9 +16,7 @@ export const config = { { id: "high-throughput", label: "High-Throughput" }, { id: "hicache", label: "HiCache + Mooncake" }, ], - nodesOptions: [ - { id: "single", label: "Single Node" }, - ], + nodesOptions: [{ id: "single", label: "Single Node" }], modelNames: { "default|bf16": "inclusionAI/Ling-3.0-flash", @@ -98,6 +94,20 @@ sgl-eval run gsm8k \\ { id: "current", label: "Inherited from base" }, { id: "off", label: "Off (greedy)" }, { id: "nextn", label: "NEXTN (built-in MTP)", flags: ["--speculative-algorithm NEXTN"] }, + { + id: "dspark", + label: "DSPARK (draft model)", + // --linear-replayssm-cache-len 32: the draft's block size 8 makes the + // verify window 9 tokens, and the KDA ReplaySSM ring must be a power + // of two >= 2x the window — the 16 default is too small and the + // server refuses to start. + flags: [ + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path inclusionAI/Ling-3.0-flash-dspark", + "--enable-linear-replayssm-spec", + "--linear-replayssm-cache-len 32", + ], + }, ], }, hicache: { @@ -126,9 +136,60 @@ sgl-eval run gsm8k \\ }, }, - cells: [ + matchDims: [ + { id: "variant", title: "Model Variant", options: [{ id: "default", label: "Ling-3.0-flash" }] }, { - match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + id: "quant", + title: "Quantization", + options: [ + { id: "bf16", label: "BF16" }, + { id: "fp8", label: "FP8" }, + { id: "int4", label: "INT4" }, + { id: "mxfp4", label: "MXFP4" }, + ], + }, + { + id: "strategy", + title: "Strategy", + options: [ + { id: "low-latency", label: "Low-Latency" }, + { id: "high-throughput", label: "High-Throughput" }, + { id: "hicache", label: "HiCache + Mooncake" }, + ], + }, + { + id: "spec", + title: "Spec Decode", + options: [ + { id: "nextn", label: "NEXTN (built-in MTP)" }, + { id: "dspark", label: "DSPARK (draft model)" }, + { id: "off", label: "Off (greedy)" }, + ], + }, + { id: "nodes", title: "Nodes", options: [{ id: "single", label: "Single Node" }] }, + ], + + // DSPARK twins of the low-latency cells: same shape, NEXTN swapped for the + // external-draft path. --linear-replayssm-cache-len 32 because the draft's + // block size 8 needs a power-of-two ring >= 2x the 9-token verify window. + // Everything lives inside this IIFE because Mintlify's snippet compiler only + // evaluates the exported expression — top-level module code is dropped. + cells: (() => { + const DSPARK_FLAGS = [ + "--speculative-algorithm DSPARK", + "--speculative-draft-model-path inclusionAI/Ling-3.0-flash-dspark", + "--enable-linear-replayssm-spec", + "--linear-replayssm-cache-len 32", + ]; + const dsparkTwin = (cell, verified) => ({ + ...cell, + verified, + match: { ...cell.match, spec: "dspark" }, + flags: cell.flags.flatMap((f) => (f === "--speculative-algorithm NEXTN" ? DSPARK_FLAGS : [f])), + }); + const lowLatencyCells = [ + { + match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -140,7 +201,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -152,7 +213,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h800", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + match: { hw: "h800", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -164,7 +225,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h100", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + match: { hw: "h100", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -176,7 +237,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -188,7 +249,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", nodes: "single" }, + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -200,7 +261,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -213,7 +274,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -226,7 +287,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h800", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "h800", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -239,7 +300,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h100", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "h100", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -252,7 +313,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -265,7 +326,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", nodes: "single" }, + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "low-latency", spec: "nextn", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -277,8 +338,14 @@ sgl-eval run gsm8k \\ "--port {{PORT}}", ], }, +]; + return [ + ...lowLatencyCells.flatMap((c) => [ + c, + dsparkTwin(c, c.match.hw === "b200" && c.match.quant === "bf16"), + ]), { - match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -289,7 +356,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -300,7 +367,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h800", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h800", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -311,7 +378,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h100", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h100", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -322,7 +389,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -333,7 +400,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" }, + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -344,7 +411,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -356,7 +423,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -368,7 +435,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h800", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h800", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -380,7 +447,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h100", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h100", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false, flags: [ "--model-path {{MODEL_NAME}}", @@ -392,7 +459,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -404,7 +471,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" }, + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -416,7 +483,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -429,7 +496,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -442,7 +509,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -456,7 +523,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: true, flags: [ "--model-path {{MODEL_NAME}}", @@ -474,7 +541,7 @@ sgl-eval run gsm8k \\ // Hybrid KDA must pass prefix keys to Mooncake; otherwise storage writes are empty. // Cold uncached extends above chunked_prefill_size skip write-through for that influx. { - match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "hicache", spec: "nextn", nodes: "single" }, verified: true, env: [ "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", @@ -502,7 +569,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "h200", variant: "default", quant: "fp8", strategy: "hicache", nodes: "single" }, + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "hicache", spec: "nextn", nodes: "single" }, verified: false, env: [ "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", @@ -531,7 +598,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "hicache", nodes: "single" }, + match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "hicache", spec: "nextn", nodes: "single" }, verified: false, env: [ "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", @@ -559,7 +626,7 @@ sgl-eval run gsm8k \\ ], }, { - match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "hicache", nodes: "single" }, + match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "hicache", spec: "nextn", nodes: "single" }, verified: false, env: [ "MOONCAKE_MASTER={{MOONCAKE_MASTER}}", @@ -587,5 +654,6 @@ sgl-eval run gsm8k \\ "--port {{PORT}}", ], }, - ], + ]; + })(), };