minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Docs] Add Ling-3.0-tiny INT4 recipes (#34395)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-08-12 03:16:03 +08:00
committed by GitHub
parent 93c1bff1d4
commit d5d41d07ed
3 changed files with 92 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/src/snippets/configs/inclusionAI/ling-3.0-tiny-benchmarks.jsx
+27 -4
View File
@@ -1,7 +1,6 @@
// Measured on lmsysorg/sglang:dev-Ling-3.0-tiny, 1× H200. TTFT/TPOT are P50
// (median) from sglang.bench_serving (random ISL 8192 / OSL 1024, --flush-cache);
// tokens_per_sec_per_gpu = output tok/s × (isl+osl)/osl. Accuracy from sgl-eval
// full GSM8K (1319).
// TTFT/TPOT are P50. INT4 uses 80 exact ISL 8192 / OSL 1024 requests with
// --flush-cache; BF16/FP8 retain their original published measurements.
// Accuracy is full GSM8K (1319).
export const benchmarks = [
{
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
@@ -25,4 +24,28 @@ export const benchmarks = [
],
accuracy: { gsm8k_pct: 94.69 },
},
{
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
sglang_version: "PR #33561 @ 8ba213fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 90.31, tpot_ms: 1.96, tokens_per_sec_per_gpu: 4398 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 840.04, tpot_ms: 3.55, tokens_per_sec_per_gpu: 32958 },
],
accuracy: { gsm8k_pct: 94.54 },
notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 36 shapes through batch 256.",
},
{
match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
sglang_version: "PR #33561 @ 8ba213fc",
speed: [
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
ttft_ms: 305.67, tpot_ms: 6.33, tokens_per_sec_per_gpu: 1359 },
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
ttft_ms: 2634.12, tpot_ms: 16.04, tokens_per_sec_per_gpu: 7730 },
],
accuracy: { gsm8k_pct: 94.54 },
notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 52 shapes through batch 512. Triton WNA16 used untuned default E=128,N=256 configs.",
},
];
docs/src/snippets/configs/inclusionAI/ling-3.0-tiny.jsx
+60 -1
View File
@@ -10,6 +10,7 @@ export const config = {
quantizations: [
{ id: "bf16", label: "BF16" },
{ id: "fp8", label: "FP8" },
{ id: "int4", label: "INT4" },
],
strategies: [
{ id: "high-throughput", label: "High-Throughput" },
@@ -21,6 +22,7 @@ export const config = {
modelNames: {
"default|bf16": "inclusionAI/Ling-3.0-tiny",
"default|fp8": "inclusionAI/Ling-3.0-tiny-fp8",
"default|int4": "inclusionAI/Ling-3.0-tiny-int4",
},
placeholders: {
@@ -51,13 +53,16 @@ export const config = {
--model {{MODEL_NAME}} \\
--dataset-name {{DATASET}} \\
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
--random-range-ratio 1 \\
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
--flush-cache`,
accuracy: {
gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
sgl-eval run gsm8k \\
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
--num-threads 32`,
--num-threads 32 \\
--temperature 1.0 --top-p 0.95 \\
--thinking`,
},
},
@@ -187,5 +192,59 @@ sgl-eval run gsm8k \\
"--port {{PORT}}",
],
},
{
match: { hw: "h20-3e", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: false,
flags: [
"--model-path {{MODEL_NAME}}",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
flags: [
"--model-path {{MODEL_NAME}}",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "h800", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: false,
flags: [
"--model-path {{MODEL_NAME}}",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "h100", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: false,
flags: [
"--model-path {{MODEL_NAME}}",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: true,
flags: [
"--model-path {{MODEL_NAME}}",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
{
match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
verified: false,
flags: [
"--model-path {{MODEL_NAME}}",
"--host {{HOST_IP}}",
"--port {{PORT}}",
],
},
],
};