diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx index d50fcf958..e9f571c7a 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx @@ -66,7 +66,7 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports - Both the chat template and the `ling3` reasoning parser default to thinking on. A single request can turn it off with `"chat_template_kwargs": {"enable_thinking": false}` (see §3.1). - The BF16/FP8 recipes use `--mem-fraction-static 0.8`; INT4/MXFP4 use `0.85`. These values reserve the headroom used by the validated graph-enabled runs. - The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes default to it via the **Spec Decode** selector; toggle it in the **Speculative Decoding** card of the [Playground](#playground). -- DSPARK is the alternative speculative path: it drafts with the external [inclusionAI/Ling-3.0-flash-dspark](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark) checkpoint instead of the built-in MTP layer. Pick **DSPARK** in the Deploy panel's **Spec Decode** selector (also available in the [Playground](#playground)); it composes `--speculative-algorithm DSPARK`, the draft path, and `--enable-linear-replayssm-spec`, the KDA verify path that folds per-draft intermediate states onto a fixed ring instead of snapshotting full states. The draft block size auto-infers from the checkpoint; tune it with `--speculative-dspark-block-size`. The draft's block size of 8 makes the verify window 9 tokens, and the KDA ReplaySSM ring must be a power of two at least twice the window — the recipe therefore pins `--linear-replayssm-cache-len 32`, since the 16 default fails startup validation. Validated on BF16, 4×B200 TP4, thinking on: full GSM8K 96.66% with a 99.77% stop rate (NEXTN on the same setup: 96.44% / 99.62%); per-cell speed and accuracy numbers sit under the B200 low-latency DSPARK cell in the Deploy panel. +- DSPARK is the alternative speculative path: it drafts with the external [inclusionAI/Ling-3.0-flash-dspark](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark) checkpoint instead of the built-in MTP layer. Pick **DSPARK** in the Deploy panel's **Spec Decode** selector (also available in the [Playground](#playground)); it composes `--speculative-algorithm DSPARK`, the draft path, and `--enable-linear-replayssm-spec`, the KDA verify path that folds per-draft intermediate states onto a fixed ring instead of snapshotting full states. The draft block size auto-infers from the checkpoint; tune it with `--speculative-dspark-block-size`. The draft's block size of 8 makes the verify window 9 tokens, and the KDA ReplaySSM ring must be a power of two at least twice the window — the recipe therefore pins `--linear-replayssm-cache-len 32`, since the 16 default fails startup validation. MXFP4 requires `--moe-runner-backend flashinfer_mxfp4`; the mixed checkpoint does not auto-select it. - Native context is 256K; SGLang reads it from the checkpoint's `max_position_embeddings`, so no `--context-length` flag is needed. - The **HiCache** card in the [Playground](#playground) exposes the validated Mooncake L3 path. It adds the hybrid-KDA scheduler and prefix-key settings together; see §3.3 for the required services. diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx index 3904b2612..dbde50567 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx @@ -16,6 +16,12 @@ export const benchmarks = [ sglang_version: "PR #33561 @ c5071ded", accuracy: { gsm8k_pct: 96.59 }, }, + { + match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "dspark", nodes: "single" }, + sglang_version: "PR #33561 @ 76a3e673", + accuracy: { gsm8k_pct: 96.36 }, + notes: "Full GSM8K stop rate 99.62%; accept length ~4.5-5.1.", + }, { // Rejected by the full GSM8K gate at request 1319: a no-EOS runaway generated // >33k tokens. Recipe stays `verified: false` in ling-3.0-flash.jsx. @@ -37,6 +43,24 @@ export const benchmarks = [ sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 96.51 }, }, + { + match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", spec: "dspark", nodes: "single" }, + sglang_version: "PR #33561 @ 76a3e673", + accuracy: { gsm8k_pct: 96.13 }, + notes: "Full GSM8K stop rate 99.55%; accept length ~3.3-4.4.", + }, + { + match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", spec: "dspark", nodes: "single" }, + sglang_version: "PR #33561 @ 76a3e673", + accuracy: { gsm8k_pct: 96.36 }, + notes: "Full GSM8K stop rate 99.62%; accept length ~4.6-4.9.", + }, + { + match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "low-latency", spec: "dspark", nodes: "single" }, + sglang_version: "PR #33561 @ 76a3e673", + accuracy: { gsm8k_pct: 96.29 }, + notes: "Full GSM8K stop rate 99.85%; accept length ~3.8-5.9.", + }, { match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", spec: "off", nodes: "single" }, sglang_version: "PR #33561 @ e1a24a18", diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx index 8845b0b37..3bfa74ccd 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx @@ -187,6 +187,9 @@ sgl-eval run gsm8k \\ match: { ...cell.match, spec: "dspark" }, flags: cell.flags.flatMap((f) => (f === "--speculative-algorithm NEXTN" ? DSPARK_FLAGS : [f])), }); + // hw|quant pairs with a measured full-GSM8K DSPARK run; see the mdx + // DSPARK tip for scores and stop rates. + const DSPARK_VERIFIED = new Set(["b200|bf16", "h200|bf16", "h200|fp8"]); const lowLatencyCells = [ { match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" }, @@ -342,8 +345,35 @@ sgl-eval run gsm8k \\ return [ ...lowLatencyCells.flatMap((c) => [ c, - dsparkTwin(c, c.match.hw === "b200" && c.match.quant === "bf16"), + dsparkTwin(c, DSPARK_VERIFIED.has(`${c.match.hw}|${c.match.quant}`)), ]), + // INT4/MXFP4 have no NEXTN low-latency cell (their twin source), so the + // DSPARK legs validated on 4xH200 are listed directly. + { + match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", spec: "dspark", nodes: "single" }, + verified: true, + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + ...DSPARK_FLAGS, + "--mem-fraction-static 0.85", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "low-latency", spec: "dspark", nodes: "single" }, + verified: true, + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--moe-runner-backend flashinfer_mxfp4", + ...DSPARK_FLAGS, + "--mem-fraction-static 0.85", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, { match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" }, verified: false,