[Docs] Add Ling-3.0-flash INT4 and MXFP4 recipes (#34363)

2026-08-11 00:35:27 -07:00
parent 9d4be40124
commit 1c06c160f9
3 changed files with 117 additions and 8 deletions
@@ -1,6 +1,6 @@
 ---
 title: Ling-3.0-flash
-description: "Deploy Ling-3.0-flash with SGLang — 124B total / 5.1B active hybrid KDA + MLA MoE in BF16 or FP8, with thinking mode, Ling3 parsers, and NEXTN speculative decoding."
+description: "Deploy Ling-3.0-flash with SGLang — a 124B hybrid KDA + MLA MoE in BF16, FP8, INT4, or MXFP4 on Hopper and Blackwell GPUs."
 tag: NEW
 ---

@@ -48,6 +48,8 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports

 - **BF16**: [inclusionAI/Ling-3.0-flash](https://huggingface.co/inclusionAI/Ling-3.0-flash) — 124B total / 5.1B active
 - **FP8** (blockwise E4M3): [inclusionAI/Ling-3.0-flash-fp8](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp8)
+- **INT4** (compressed-tensors W4A16): [inclusionAI/Ling-3.0-flash-int4](https://huggingface.co/inclusionAI/Ling-3.0-flash-int4)
+- **MXFP4**: [inclusionAI/Ling-3.0-flash-fp4](https://huggingface.co/inclusionAI/Ling-3.0-flash-fp4)

 **License:** MIT

@@ -57,9 +59,11 @@ It is a hybrid-reasoning model with thinking enabled by default, and it supports

 - BF16 tensor parallelism follows the GPU: `--tp 4` on 141 GB-class cards (H20-3e, H200) and 4-GPU Blackwell nodes (B200, GB300); `--tp 8` on 80 GB cards (H100, H800).
 - The FP8 recipes pair `--tp` with a matching `--ep-size` (`--tp 4 --ep-size 4` on 4-GPU nodes, `--tp 8 --ep-size 8` on H100/H800). The checkpoint uses blockwise (128×128) E4M3 expert weights, so a pure tensor-parallel shard must satisfy `(768 / TP) % 128 == 0` — only TP2 qualifies; expert parallelism splits experts whole instead of by column, which lifts that restriction and uses the full node. SGLang detects the quantization format from the checkpoint's `quantization_config`, so no explicit quantization flag is needed.
+- INT4 uses compressed-tensors W4A16 experts. With the default MoE runner, SGLang selects Marlin on H200 and the graph-compatible Triton implementation on Blackwell; do not force a backend in the launch command.
+- MXFP4 uses the native FlashInfer runner on H200 and B200. The checkpoint mixes MXFP4 routed experts with block-FP8 dense and shared projections; on B200, `--fp8-gemm-backend triton` avoids an unsupported FlashInfer FP8 tactic while the routed experts remain native MXFP4.
 - `--reasoning-parser ling3` and `--tool-call-parser ling3` enable Ling-3.0-specific reasoning and structured tool-call parsing; toggle them in the **Parsers** card of the [Playground](#playground).
 - Both the chat template and the `ling3` reasoning parser default to thinking on. A single request can turn it off with `"chat_template_kwargs": {"enable_thinking": false}` (see §3.1).
- `--mem-fraction-static 0.8` reserves headroom for CUDA graphs and concurrent decoding; with the default allocation the NEXTN recipes can OOM under concurrent requests (e.g. a 32-thread GSM8K run).
+- The BF16/FP8 recipes use `--mem-fraction-static 0.8`; INT4/MXFP4 use `0.85`. These values reserve the headroom used by the validated graph-enabled runs.
 - The checkpoint ships a built-in MTP layer (`num_nextn_predict_layers: 1`); enable it with `--speculative-algorithm NEXTN` — no separate draft model is needed. The Low-Latency recipes have it on; toggle it in the **Speculative Decoding** card of the [Playground](#playground).
 - Native context is 256K; SGLang reads it from the checkpoint's `max_position_embeddings`, so no `--context-length` flag is needed.
 - The **HiCache** card in the [Playground](#playground) exposes the validated Mooncake L3 path. It adds the hybrid-KDA scheduler and prefix-key settings together; see §3.3 for the required services.
@@ -1,12 +1,9 @@
 // Ling-3.0-flash per-cell benchmark numbers, keyed by the same `match` tuple as
 // ling-3.0-flash.jsx cells. See _deployment.jsx for the speed/accuracy schema.
 //
-// Accuracy harness (one harness for the whole GSM8K column, per
-// config.benchmarkCommands.accuracy): sgl-eval run gsm8k, full 1319 questions,
-// --num-threads 32. Every filled entry below also recorded 100% stop /
-// 0% truncated / 0% error.
-//
-// Speed numbers are not measured yet — entries carry accuracy only.
+// Accuracy uses sgl-eval full GSM8K (1319 questions). Speed uses 80 exact-length
+// random requests (ISL 8192 / OSL 1024, --random-range-ratio 1, --flush-cache).
+// TTFT/TPOT are P50; tokens_per_sec_per_gpu is total (input + output) tok/s/GPU.
 //
 // Cells with no entry (H20-3e / H800 / H100, both quantizations) had no matching
 // allocation and were never gated.
@@ -40,6 +37,30 @@ export const benchmarks = [
    sglang_version: "PR #33561 @ e57e030b",
    accuracy: { gsm8k_pct: 96.51 },
  },
+  {
+    match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ e1a24a18",
+    speed: [
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 205.96, tpot_ms: 3.68, tokens_per_sec_per_gpu: 1159 },
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 1891.74, tpot_ms: 8.16, tokens_per_sec_per_gpu: 7184 },
+    ],
+    accuracy: { gsm8k_pct: 95.30 },
+    notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 32 shapes through batch 218.",
+  },
+  {
+    match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ e1a24a18",
+    speed: [
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 305.23, tpot_ms: 6.05, tokens_per_sec_per_gpu: 709 },
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 2725.52, tpot_ms: 11.17, tokens_per_sec_per_gpu: 5215 },
+    ],
+    accuracy: { gsm8k_pct: 96.29 },
+    notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 33 shapes through batch 227.",
+  },

  // ====================================================================
  // H200 + HiCache (Mooncake tiered cache)
@@ -77,6 +98,30 @@ export const benchmarks = [
    sglang_version: "PR #33561 @ e57e030b",
    accuracy: { gsm8k_pct: 97.04 },
  },
+  {
+    match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ e1a24a18",
+    speed: [
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 641.41, tpot_ms: 8.72, tokens_per_sec_per_gpu: 482 },
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 6202.62, tpot_ms: 48.01, tokens_per_sec_per_gpu: 1344 },
+    ],
+    accuracy: { gsm8k_pct: 96.74 },
+    notes: "Full GSM8K stop rate 99.70%; default decode CUDA Graph captured 40 shapes through batch 305.",
+  },
+  {
+    match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
+    sglang_version: "PR #33561 @ e1a24a18",
+    speed: [
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
+        ttft_ms: 127.97, tpot_ms: 5.60, tokens_per_sec_per_gpu: 785 },
+      { workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
+        ttft_ms: 1117.16, tpot_ms: 7.98, tokens_per_sec_per_gpu: 7933 },
+    ],
+    accuracy: { gsm8k_pct: 96.29 },
+    notes: "Full GSM8K stop rate 99.62%; default decode CUDA Graph captured 40 shapes through batch 314.",
+  },

  // ====================================================================
  // GB300 + BF16 (TP4)
@@ -10,6 +10,8 @@ export const config = {
  quantizations: [
    { id: "bf16", label: "BF16" },
    { id: "fp8", label: "FP8" },
+    { id: "int4", label: "INT4" },
+    { id: "mxfp4", label: "MXFP4" },
  ],
  strategies: [
    { id: "low-latency", label: "Low-Latency" },
@@ -23,6 +25,8 @@ export const config = {
  modelNames: {
    "default|bf16": "inclusionAI/Ling-3.0-flash",
    "default|fp8": "inclusionAI/Ling-3.0-flash-fp8",
+    "default|int4": "inclusionAI/Ling-3.0-flash-int4",
+    "default|mxfp4": "inclusionAI/Ling-3.0-flash-fp4",
  },

  placeholders: {
@@ -58,6 +62,7 @@ export const config = {
  --model {{MODEL_NAME}} \\
  --dataset-name {{DATASET}} \\
  --random-input-len {{ISL}} --random-output-len {{OSL}} \\
+  --random-range-ratio 1 \\
  --num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
  --flush-cache`,
    accuracy: {
@@ -410,6 +415,61 @@ sgl-eval run gsm8k \\
        "--port {{PORT}}",
      ],
    },
+    {
+      match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 2",
+        "--mem-fraction-static 0.85",
+        "--tool-call-parser ling3",
+        "--reasoning-parser ling3",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 2",
+        "--mem-fraction-static 0.85",
+        "--tool-call-parser ling3",
+        "--reasoning-parser ling3",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 2",
+        "--moe-runner-backend flashinfer_mxfp4",
+        "--mem-fraction-static 0.85",
+        "--tool-call-parser ling3",
+        "--reasoning-parser ling3",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },
+    {
+      match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
+      verified: true,
+      flags: [
+        "--model-path {{MODEL_NAME}}",
+        "--tp 2",
+        "--moe-runner-backend flashinfer_mxfp4",
+        "--fp8-gemm-backend triton",
+        "--mem-fraction-static 0.85",
+        "--tool-call-parser ling3",
+        "--reasoning-parser ling3",
+        "--host {{HOST_IP}}",
+        "--port {{PORT}}",
+      ],
+    },

    // Hybrid KDA must pass prefix keys to Mooncake; otherwise storage writes are empty.
    // Cold uncached extends above chunked_prefill_size skip write-through for that influx.