docs: mark Ling-3.0-flash DSPARK verified for all four quantizations on H200 (#36204)
This commit is contained in:
Xinyuan Tong
@@ -66,7 +66,7 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports
|
||||
- Both the chat template and the `ling3` reasoning parser default to thinking on. A single request can turn it off with `"chat_template_kwargs": {"enable_thinking": false}` (see §3.1).
|
||||
- The BF16/FP8 recipes use `--mem-fraction-static 0.8`; INT4/MXFP4 use `0.85`. These values reserve the headroom used by the validated graph-enabled runs.
|
||||
- The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes default to it via the **Spec Decode** selector; toggle it in the **Speculative Decoding** card of the [Playground](#playground).
|
||||
- DSPARK is the alternative speculative path: it drafts with the external [inclusionAI/Ling-3.0-flash-dspark](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark) checkpoint instead of the built-in MTP layer. Pick **DSPARK** in the Deploy panel's **Spec Decode** selector (also available in the [Playground](#playground)); it composes `--speculative-algorithm DSPARK`, the draft path, and `--enable-linear-replayssm-spec`, the KDA verify path that folds per-draft intermediate states onto a fixed ring instead of snapshotting full states. The draft block size auto-infers from the checkpoint; tune it with `--speculative-dspark-block-size`. The draft's block size of 8 makes the verify window 9 tokens, and the KDA ReplaySSM ring must be a power of two at least twice the window — the recipe therefore pins `--linear-replayssm-cache-len 32`, since the 16 default fails startup validation. Validated on BF16, 4×B200 TP4, thinking on: full GSM8K 96.66% with a 99.77% stop rate (NEXTN on the same setup: 96.44% / 99.62%); per-cell speed and accuracy numbers sit under the B200 low-latency DSPARK cell in the Deploy panel.
|
||||
- DSPARK is the alternative speculative path: it drafts with the external [inclusionAI/Ling-3.0-flash-dspark](https://huggingface.co/inclusionAI/Ling-3.0-flash-dspark) checkpoint instead of the built-in MTP layer. Pick **DSPARK** in the Deploy panel's **Spec Decode** selector (also available in the [Playground](#playground)); it composes `--speculative-algorithm DSPARK`, the draft path, and `--enable-linear-replayssm-spec`, the KDA verify path that folds per-draft intermediate states onto a fixed ring instead of snapshotting full states. The draft block size auto-infers from the checkpoint; tune it with `--speculative-dspark-block-size`. The draft's block size of 8 makes the verify window 9 tokens, and the KDA ReplaySSM ring must be a power of two at least twice the window — the recipe therefore pins `--linear-replayssm-cache-len 32`, since the 16 default fails startup validation. MXFP4 requires `--moe-runner-backend flashinfer_mxfp4`; the mixed checkpoint does not auto-select it.
|
||||
- Native context is 256K; SGLang reads it from the checkpoint's `max_position_embeddings`, so no `--context-length` flag is needed.
|
||||
- The **HiCache** card in the [Playground](#playground) exposes the validated Mooncake L3 path. It adds the hybrid-KDA scheduler and prefix-key settings together; see §3.3 for the required services.
|
||||
|
||||
|
||||
@@ -16,6 +16,12 @@ export const benchmarks = [
|
||||
sglang_version: "PR #33561 @ c5071ded",
|
||||
accuracy: { gsm8k_pct: 96.59 },
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "low-latency", spec: "dspark", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ 76a3e673",
|
||||
accuracy: { gsm8k_pct: 96.36 },
|
||||
notes: "Full GSM8K stop rate 99.62%; accept length ~4.5-5.1.",
|
||||
},
|
||||
{
|
||||
// Rejected by the full GSM8K gate at request 1319: a no-EOS runaway generated
|
||||
// >33k tokens. Recipe stays `verified: false` in ling-3.0-flash.jsx.
|
||||
@@ -37,6 +43,24 @@ export const benchmarks = [
|
||||
sglang_version: "PR #33561 @ e57e030b",
|
||||
accuracy: { gsm8k_pct: 96.51 },
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "low-latency", spec: "dspark", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ 76a3e673",
|
||||
accuracy: { gsm8k_pct: 96.13 },
|
||||
notes: "Full GSM8K stop rate 99.55%; accept length ~3.3-4.4.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", spec: "dspark", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ 76a3e673",
|
||||
accuracy: { gsm8k_pct: 96.36 },
|
||||
notes: "Full GSM8K stop rate 99.62%; accept length ~4.6-4.9.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "low-latency", spec: "dspark", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ 76a3e673",
|
||||
accuracy: { gsm8k_pct: 96.29 },
|
||||
notes: "Full GSM8K stop rate 99.85%; accept length ~3.8-5.9.",
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", spec: "off", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ e1a24a18",
|
||||
|
||||
@@ -187,6 +187,9 @@ sgl-eval run gsm8k \\
|
||||
match: { ...cell.match, spec: "dspark" },
|
||||
flags: cell.flags.flatMap((f) => (f === "--speculative-algorithm NEXTN" ? DSPARK_FLAGS : [f])),
|
||||
});
|
||||
// hw|quant pairs with a measured full-GSM8K DSPARK run; see the mdx
|
||||
// DSPARK tip for scores and stop rates.
|
||||
const DSPARK_VERIFIED = new Set(["b200|bf16", "h200|bf16", "h200|fp8"]);
|
||||
const lowLatencyCells = [
|
||||
{
|
||||
match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "low-latency", spec: "nextn", nodes: "single" },
|
||||
@@ -342,8 +345,35 @@ sgl-eval run gsm8k \\
|
||||
return [
|
||||
...lowLatencyCells.flatMap((c) => [
|
||||
c,
|
||||
dsparkTwin(c, c.match.hw === "b200" && c.match.quant === "bf16"),
|
||||
dsparkTwin(c, DSPARK_VERIFIED.has(`${c.match.hw}|${c.match.quant}`)),
|
||||
]),
|
||||
// INT4/MXFP4 have no NEXTN low-latency cell (their twin source), so the
|
||||
// DSPARK legs validated on 4xH200 are listed directly.
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "int4", strategy: "low-latency", spec: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 2",
|
||||
...DSPARK_FLAGS,
|
||||
"--mem-fraction-static 0.85",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "low-latency", spec: "dspark", nodes: "single" },
|
||||
verified: true,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--tp 2",
|
||||
"--moe-runner-backend flashinfer_mxfp4",
|
||||
...DSPARK_FLAGS,
|
||||
"--mem-fraction-static 0.85",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", spec: "off", nodes: "single" },
|
||||
verified: false,
|
||||
|
||||
Reference in New Issue
Block a user