From 1c06c160f99c35c910014c7c6c339078d7d9adea Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:35:27 +0800 Subject: [PATCH] [Docs] Add Ling-3.0-flash INT4 and MXFP4 recipes (#34363) --- .../InclusionAI/Ling-3.0-flash.mdx | 8 ++- .../inclusionAI/ling-3.0-flash-benchmarks.jsx | 57 ++++++++++++++++-- .../configs/inclusionAI/ling-3.0-flash.jsx | 60 +++++++++++++++++++ 3 files changed, 117 insertions(+), 8 deletions(-) diff --git a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx index 91ca0b36f..c1076867b 100644 --- a/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx +++ b/docs/cookbook/autoregressive/InclusionAI/Ling-3.0-flash.mdx @@ -1,6 +1,6 @@ --- title: Ling-3.0-flash -description: "Deploy Ling-3.0-flash with SGLang — 124B total / 5.1B active hybrid KDA + MLA MoE in BF16 or FP8, with thinking mode, Ling3 parsers, and NEXTN speculative decoding." +description: "Deploy Ling-3.0-flash with SGLang — a 124B hybrid KDA + MLA MoE in BF16, FP8, INT4, or MXFP4 on Hopper and Blackwell GPUs." tag: NEW --- @@ -48,6 +48,8 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports - **BF16**: [inclusionAI/Ling-3.0-flash](https://huggingface.co/inclusionAI/Ling-3.0-flash) — 124B total / 5.1B active - **FP8** (blockwise E4M3): [inclusionAI/Ling-3.0-flash-fp8](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp8) +- **INT4** (compressed-tensors W4A16): [inclusionAI/Ling-3.0-flash-int4](https://huggingface.co/inclusionAI/Ling-3.0-flash-int4) +- **MXFP4**: [inclusionAI/Ling-3.0-flash-fp4](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp4) **License:** MIT @@ -57,9 +59,11 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports - BF16 tensor parallelism follows the GPU: `--tp 4` on 141 GB-class cards (H20-3e, H200) and 4-GPU Blackwell nodes (B200, GB300); `--tp 8` on 80 GB cards (H100, H800). - The FP8 recipes pair `--tp` with a matching `--ep-size` (`--tp 4 --ep-size 4` on 4-GPU nodes, `--tp 8 --ep-size 8` on H100/H800). The checkpoint uses blockwise (128×128) E4M3 expert weights, so a pure tensor-parallel shard must satisfy `(768 / TP) % 128 == 0` — only TP2 qualifies; expert parallelism splits experts whole instead of by column, which lifts that restriction and uses the full node. SGLang detects the quantization format from the checkpoint's `quantization_config`, so no explicit quantization flag is needed. +- INT4 uses compressed-tensors W4A16 experts. With the default MoE runner, SGLang selects Marlin on H200 and the graph-compatible Triton implementation on Blackwell; do not force a backend in the launch command. +- MXFP4 uses the native FlashInfer runner on H200 and B200. The checkpoint mixes MXFP4 routed experts with block-FP8 dense and shared projections; on B200, `--fp8-gemm-backend triton` avoids an unsupported FlashInfer FP8 tactic while the routed experts remain native MXFP4. - `--reasoning-parser ling3` and `--tool-call-parser ling3` enable Ling-3.0-specific reasoning and structured tool-call parsing; toggle them in the **Parsers** card of the [Playground](#playground). - Both the chat template and the `ling3` reasoning parser default to thinking on. A single request can turn it off with `"chat_template_kwargs": {"enable_thinking": false}` (see §3.1). -- `--mem-fraction-static 0.8` reserves headroom for CUDA graphs and concurrent decoding; with the default allocation the NEXTN recipes can OOM under concurrent requests (e.g. a 32-thread GSM8K run). +- The BF16/FP8 recipes use `--mem-fraction-static 0.8`; INT4/MXFP4 use `0.85`. These values reserve the headroom used by the validated graph-enabled runs. - The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes have it on; toggle it in the **Speculative Decoding** card of the [Playground](#playground). - Native context is 256K; SGLang reads it from the checkpoint's `max_position_embeddings`, so no `--context-length` flag is needed. - The **HiCache** card in the [Playground](#playground) exposes the validated Mooncake L3 path. It adds the hybrid-KDA scheduler and prefix-key settings together; see §3.3 for the required services. diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx index 50fa6f4ad..e4d7a1e7a 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx @@ -1,12 +1,9 @@ // Ling-3.0-flash per-cell benchmark numbers, keyed by the same `match` tuple as // ling-3.0-flash.jsx cells. See _deployment.jsx for the speed/accuracy schema. // -// Accuracy harness (one harness for the whole GSM8K column, per -// config.benchmarkCommands.accuracy): sgl-eval run gsm8k, full 1319 questions, -// --num-threads 32. Every filled entry below also recorded 100% stop / -// 0% truncated / 0% error. -// -// Speed numbers are not measured yet — entries carry accuracy only. +// Accuracy uses sgl-eval full GSM8K (1319 questions). Speed uses 80 exact-length +// random requests (ISL 8192 / OSL 1024, --random-range-ratio 1, --flush-cache). +// TTFT/TPOT are P50; tokens_per_sec_per_gpu is total (input + output) tok/s/GPU. // // Cells with no entry (H20-3e / H800 / H100, both quantizations) had no matching // allocation and were never gated. @@ -40,6 +37,30 @@ export const benchmarks = [ sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 96.51 }, }, + { + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "PR #33561 @ e1a24a18", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 205.96, tpot_ms: 3.68, tokens_per_sec_per_gpu: 1159 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 1891.74, tpot_ms: 8.16, tokens_per_sec_per_gpu: 7184 }, + ], + accuracy: { gsm8k_pct: 95.30 }, + notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 32 shapes through batch 218.", + }, + { + match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "PR #33561 @ e1a24a18", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 305.23, tpot_ms: 6.05, tokens_per_sec_per_gpu: 709 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 2725.52, tpot_ms: 11.17, tokens_per_sec_per_gpu: 5215 }, + ], + accuracy: { gsm8k_pct: 96.29 }, + notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 33 shapes through batch 227.", + }, // ==================================================================== // H200 + HiCache (Mooncake tiered cache) @@ -77,6 +98,30 @@ export const benchmarks = [ sglang_version: "PR #33561 @ e57e030b", accuracy: { gsm8k_pct: 97.04 }, }, + { + match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "PR #33561 @ e1a24a18", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 641.41, tpot_ms: 8.72, tokens_per_sec_per_gpu: 482 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 6202.62, tpot_ms: 48.01, tokens_per_sec_per_gpu: 1344 }, + ], + accuracy: { gsm8k_pct: 96.74 }, + notes: "Full GSM8K stop rate 99.70%; default decode CUDA Graph captured 40 shapes through batch 305.", + }, + { + match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + sglang_version: "PR #33561 @ e1a24a18", + speed: [ + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 }, + ttft_ms: 127.97, tpot_ms: 5.60, tokens_per_sec_per_gpu: 785 }, + { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 }, + ttft_ms: 1117.16, tpot_ms: 7.98, tokens_per_sec_per_gpu: 7933 }, + ], + accuracy: { gsm8k_pct: 96.29 }, + notes: "Full GSM8K stop rate 99.62%; default decode CUDA Graph captured 40 shapes through batch 314.", + }, // ==================================================================== // GB300 + BF16 (TP4) diff --git a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx index e330de05f..7b9b329c8 100644 --- a/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx +++ b/docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx @@ -10,6 +10,8 @@ export const config = { quantizations: [ { id: "bf16", label: "BF16" }, { id: "fp8", label: "FP8" }, + { id: "int4", label: "INT4" }, + { id: "mxfp4", label: "MXFP4" }, ], strategies: [ { id: "low-latency", label: "Low-Latency" }, @@ -23,6 +25,8 @@ export const config = { modelNames: { "default|bf16": "inclusionAI/Ling-3.0-flash", "default|fp8": "inclusionAI/Ling-3.0-flash-fp8", + "default|int4": "inclusionAI/Ling-3.0-flash-int4", + "default|mxfp4": "inclusionAI/Ling-3.0-flash-fp4", }, placeholders: { @@ -58,6 +62,7 @@ export const config = { --model {{MODEL_NAME}} \\ --dataset-name {{DATASET}} \\ --random-input-len {{ISL}} --random-output-len {{OSL}} \\ + --random-range-ratio 1 \\ --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\ --flush-cache`, accuracy: { @@ -410,6 +415,61 @@ sgl-eval run gsm8k \\ "--port {{PORT}}", ], }, + { + match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--mem-fraction-static 0.85", + "--tool-call-parser ling3", + "--reasoning-parser ling3", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" }, + verified: true, + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--mem-fraction-static 0.85", + "--tool-call-parser ling3", + "--reasoning-parser ling3", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--moe-runner-backend flashinfer_mxfp4", + "--mem-fraction-static 0.85", + "--tool-call-parser ling3", + "--reasoning-parser ling3", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, + { + match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" }, + verified: true, + flags: [ + "--model-path {{MODEL_NAME}}", + "--tp 2", + "--moe-runner-backend flashinfer_mxfp4", + "--fp8-gemm-backend triton", + "--mem-fraction-static 0.85", + "--tool-call-parser ling3", + "--reasoning-parser ling3", + "--host {{HOST_IP}}", + "--port {{PORT}}", + ], + }, // Hybrid KDA must pass prefix keys to Mooncake; otherwise storage writes are empty. // Cold uncached extends above chunked_prefill_size skip write-through for that influx.