minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

[Docs] Add Ling-3.0-flash INT4 and MXFP4 recipes (#34363)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-08-11 00:35:27 -07:00
committed by GitHub
parent 9d4be40124
commit 1c06c160f9
3 changed files with 117 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/snippets/configs/inclusionAI/ling-3.0-flash-benchmarks.jsx
+51 -6
View File
@@ -1,12 +1,9 @@
// Ling-3.0-flash per-cell benchmark numbers, keyed by the same `match` tuple as
// ling-3.0-flash.jsx cells. See _deployment.jsx for the speed/accuracy schema.
//
// Accuracy harness (one harness for the whole GSM8K column, per
// config.benchmarkCommands.accuracy): sgl-eval run gsm8k, full 1319 questions,
// --num-threads 32. Every filled entry below also recorded 100% stop /
// 0% truncated / 0% error.
//
// Speed numbers are not measured yet — entries carry accuracy only.
// Accuracy uses sgl-eval full GSM8K (1319 questions). Speed uses 80 exact-length
// random requests (ISL 8192 / OSL 1024, --random-range-ratio 1, --flush-cache).
// TTFT/TPOT are P50; tokens_per_sec_per_gpu is total (input + output) tok/s/GPU.
//
// Cells with no entry (H20-3e / H800 / H100, both quantizations) had no matching
// allocation and were never gated.
@@ -40,6 +37,30 @@ export const benchmarks = [
sglang_version: "PR #33561 @ e57e030b",
accuracy: { gsm8k_pct: 96.51 },
},
{
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
sglang_version: "PR #33561 @ e1a24a18",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 205.96, tpot_ms: 3.68, tokens_per_sec_per_gpu: 1159 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 1891.74, tpot_ms: 8.16, tokens_per_sec_per_gpu: 7184 },
],
accuracy: { gsm8k_pct: 95.30 },
notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 32 shapes through batch 218.",
},
{
match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
sglang_version: "PR #33561 @ e1a24a18",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 305.23, tpot_ms: 6.05, tokens_per_sec_per_gpu: 709 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 2725.52, tpot_ms: 11.17, tokens_per_sec_per_gpu: 5215 },
],
accuracy: { gsm8k_pct: 96.29 },
notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 33 shapes through batch 227.",
},
// ====================================================================
// H200 + HiCache (Mooncake tiered cache)
@@ -77,6 +98,30 @@ export const benchmarks = [
sglang_version: "PR #33561 @ e57e030b",
accuracy: { gsm8k_pct: 97.04 },
},
{
match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
sglang_version: "PR #33561 @ e1a24a18",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 641.41, tpot_ms: 8.72, tokens_per_sec_per_gpu: 482 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 6202.62, tpot_ms: 48.01, tokens_per_sec_per_gpu: 1344 },
],
accuracy: { gsm8k_pct: 96.74 },
notes: "Full GSM8K stop rate 99.70%; default decode CUDA Graph captured 40 shapes through batch 305.",
},
{
match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
sglang_version: "PR #33561 @ e1a24a18",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 127.97, tpot_ms: 5.60, tokens_per_sec_per_gpu: 785 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 1117.16, tpot_ms: 7.98, tokens_per_sec_per_gpu: 7933 },
],
accuracy: { gsm8k_pct: 96.29 },
notes: "Full GSM8K stop rate 99.62%; default decode CUDA Graph captured 40 shapes through batch 314.",
},
// ====================================================================
// GB300 + BF16 (TP4)
docs/src/snippets/configs/inclusionAI/ling-3.0-flash.jsx
+60
View File
@@ -10,6 +10,8 @@ export const config = {
quantizations: [
{ id: "bf16", label: "BF16" },
{ id: "fp8", label: "FP8" },
{ id: "int4", label: "INT4" },
{ id: "mxfp4", label: "MXFP4" },
],
strategies: [
{ id: "low-latency", label: "Low-Latency" },
@@ -23,6 +25,8 @@ export const config = {
modelNames: {
"default|bf16": "inclusionAI/Ling-3.0-flash",
"default|fp8": "inclusionAI/Ling-3.0-flash-fp8",
"default|int4": "inclusionAI/Ling-3.0-flash-int4",
"default|mxfp4": "inclusionAI/Ling-3.0-flash-fp4",
},
placeholders: {
@@ -58,6 +62,7 @@ export const config = {
--model {{MODEL_NAME}} \\
--dataset-name {{DATASET}} \\
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
--random-range-ratio 1 \\
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
--flush-cache`,
accuracy: {
@@ -410,6 +415,61 @@ sgl-eval run gsm8k \\
"--port {{PORT}}",
],
},
{
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 2",
"--mem-fraction-static 0.85",
"--tool-call-parser ling3",
"--reasoning-parser ling3",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 2",
"--mem-fraction-static 0.85",
"--tool-call-parser ling3",
"--reasoning-parser ling3",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "h200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
verified: true,
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 2",
"--moe-runner-backend flashinfer_mxfp4",
"--mem-fraction-static 0.85",
"--tool-call-parser ling3",
"--reasoning-parser ling3",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b200", variant: "default", quant: "mxfp4", strategy: "high-throughput", nodes: "single" },
verified: true,
flags: [
"--model-path {{MODEL_NAME}}",
"--tp 2",
"--moe-runner-backend flashinfer_mxfp4",
"--fp8-gemm-backend triton",
"--mem-fraction-static 0.85",
"--tool-call-parser ling3",
"--reasoning-parser ling3",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
// Hybrid KDA must pass prefix keys to Mooncake; otherwise storage writes are empty.
// Cold uncached extends above chunked_prefill_size skip write-through for that influx.