Cookbook: add Ling-3.0-tiny (#34283)
This commit is contained in:
Xinyuan Tong
@@ -0,0 +1,28 @@
|
||||
// Measured on lmsysorg/sglang:dev-Ling-3.0-tiny, 1× H200. TTFT/TPOT are P50
|
||||
// (median) from sglang.bench_serving (random ISL 8192 / OSL 1024, --flush-cache);
|
||||
// tokens_per_sec_per_gpu = output tok/s × (isl+osl)/osl. Accuracy from sgl-eval
|
||||
// full GSM8K (1319).
|
||||
export const benchmarks = [
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "dev-Ling-3.0-tiny",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 69.95, tpot_ms: 2.87, tokens_per_sec_per_gpu: 3002 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 78.31, tpot_ms: 5.96, tokens_per_sec_per_gpu: 22446 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 94.01 },
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "dev-Ling-3.0-tiny",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 81.78, tpot_ms: 2.82, tokens_per_sec_per_gpu: 3072 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 79.50, tpot_ms: 5.57, tokens_per_sec_per_gpu: 23738 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 94.69 },
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,191 @@
|
||||
export const config = {
|
||||
modelName: "Ling-3.0-tiny",
|
||||
|
||||
supportedHardware: ["h20-3e", "h200", "h800", "h100", "b200", "gb300"],
|
||||
groupHardware: false,
|
||||
|
||||
variants: [
|
||||
{ id: "default", label: "Ling-3.0-tiny" },
|
||||
],
|
||||
quantizations: [
|
||||
{ id: "bf16", label: "BF16" },
|
||||
{ id: "fp8", label: "FP8" },
|
||||
],
|
||||
strategies: [
|
||||
{ id: "high-throughput", label: "High-Throughput" },
|
||||
],
|
||||
nodesOptions: [
|
||||
{ id: "single", label: "Single Node" },
|
||||
],
|
||||
|
||||
modelNames: {
|
||||
"default|bf16": "inclusionAI/Ling-3.0-tiny",
|
||||
"default|fp8": "inclusionAI/Ling-3.0-tiny-fp8",
|
||||
},
|
||||
|
||||
placeholders: {
|
||||
HOST_IP: { target: "command", label: "Bind host", default: "0.0.0.0" },
|
||||
PORT: { target: "command", label: "Bind port", default: "30000" },
|
||||
HF_TOKEN: { target: "command", label: "HF token (Docker)", default: "<your-hf-token>" },
|
||||
CURL_HOST: { target: "curl", label: "Server host", default: "localhost" },
|
||||
CURL_PORT: { target: "curl", label: "Server port", default: "30000" },
|
||||
},
|
||||
|
||||
curl: `curl http://{{CURL_HOST}}:{{CURL_PORT}}/v1/chat/completions \\
|
||||
-H 'Content-Type: application/json' \\
|
||||
-d '{ "model": "{{MODEL_NAME}}", "messages": [{"role":"user","content":"What is the capital of France?"}] }'`,
|
||||
|
||||
dockerImages: {
|
||||
"h20-3e": "lmsysorg/sglang:dev-Ling-3.0-tiny",
|
||||
"h200": "lmsysorg/sglang:dev-Ling-3.0-tiny",
|
||||
"h800": "lmsysorg/sglang:dev-Ling-3.0-tiny",
|
||||
"h100": "lmsysorg/sglang:dev-Ling-3.0-tiny",
|
||||
"b200": "lmsysorg/sglang:dev-Ling-3.0-tiny",
|
||||
"gb300": "lmsysorg/sglang:dev-Ling-3.0-tiny",
|
||||
},
|
||||
|
||||
benchmarkCommands: {
|
||||
speed: `python3 -m sglang.bench_serving \\
|
||||
--backend sglang \\
|
||||
--host {{CURL_HOST}} --port {{CURL_PORT}} \\
|
||||
--model {{MODEL_NAME}} \\
|
||||
--dataset-name {{DATASET}} \\
|
||||
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
|
||||
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
|
||||
--flush-cache`,
|
||||
accuracy: {
|
||||
gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
|
||||
sgl-eval run gsm8k \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
|
||||
--num-threads 32`,
|
||||
},
|
||||
},
|
||||
|
||||
accuracyLabels: [
|
||||
["gsm8k_pct", "GSM8K", "%"],
|
||||
],
|
||||
|
||||
github: {
|
||||
cookbookModel: "inclusionAI/Ling-3.0-tiny",
|
||||
},
|
||||
|
||||
playgroundFeatures: {
|
||||
parsers: {
|
||||
items: [
|
||||
{ id: "reasoning", label: "Reasoning Parser", flag: "--reasoning-parser deepseek-r1" },
|
||||
{ id: "toolCall", label: "Tool Call Parser", flag: "--tool-call-parser glm45" },
|
||||
],
|
||||
},
|
||||
},
|
||||
|
||||
cells: [
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h20-3e", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h800", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h20-3e", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h800", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
Reference in New Issue
Block a user