[Docs] Add Ling-3.0-tiny INT4 recipes (#34395)
This commit is contained in:
Xinyuan Tong
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Ling-3.0-tiny
|
||||
description: "Deploy Ling-3.0-tiny with SGLang — a compact ~7.9B total / ~1.2B active hybrid KDA + MLA MoE in BF16 or FP8, with thinking mode and tool calling."
|
||||
description: "Deploy Ling-3.0-tiny with SGLang — a compact ~7.9B total / ~1.2B active hybrid KDA + MLA MoE in BF16, FP8, or INT4, with thinking mode and tool calling."
|
||||
tag: NEW
|
||||
---
|
||||
|
||||
@@ -46,6 +46,7 @@ It is a thinking model with chain-of-thought enabled by default, and it supports
|
||||
|
||||
- **BF16**: [inclusionAI/Ling-3.0-tiny](https://huggingface.co/inclusionAI/Ling-3.0-tiny) — ~7.9B total / ~1.2B active
|
||||
- **FP8** (blockwise E4M3): [inclusionAI/Ling-3.0-tiny-fp8](https://huggingface.co/inclusionAI/Ling-3.0-tiny-fp8)
|
||||
- **INT4** (compressed-tensors W4A16): [inclusionAI/Ling-3.0-tiny-int4](https://huggingface.co/inclusionAI/Ling-3.0-tiny-int4)
|
||||
|
||||
**License:** MIT
|
||||
|
||||
@@ -53,9 +54,10 @@ It is a thinking model with chain-of-thought enabled by default, and it supports
|
||||
|
||||
## 2. Configuration Tips
|
||||
|
||||
- At ~7.9B total / 15.8 GB in BF16 (~7.9 GB in FP8), a single GPU is plenty on every supported card. Tensor parallelism is only useful to raise aggregate KV-cache capacity for many long-context concurrent requests — add `--tp 2`/`--tp 4` to a multi-GPU serve directly.
|
||||
- Use the dedicated `lmsysorg/sglang:dev-Ling-3.0-tiny` runtime image below; it carries the `bailing_hybrid` support Ling-3.0-tiny needs.
|
||||
- At ~7.9B total / 15.8 GB in BF16 (~7.9 GB in FP8 and ~5.8 GB in INT4), a single GPU is plenty on every supported card. Tensor parallelism is only useful to raise aggregate KV-cache capacity for many long-context concurrent requests — add `--tp 2`/`--tp 4` to a multi-GPU serve directly.
|
||||
- Use the dedicated `lmsysorg/sglang:dev-Ling-3.0-tiny` runtime image; it includes the compressed-tensors Hopper and Blackwell backends that INT4 needs.
|
||||
- The FP8 checkpoint uses blockwise (128×128) E4M3 weights with dynamic activations, quantized from the BF16 model with attention projections, the dense MoE gate, and the lm_head left in higher precision. SGLang detects the format from the checkpoint's `quantization_config`, so no explicit quantization flag is needed, and the same single-GPU recipe serves it.
|
||||
- The INT4 checkpoint uses symmetric group-32 W4A16 routed experts. SGLang selects Marlin on Hopper and Triton WNA16 on Blackwell automatically; no explicit quantization or MoE backend flag is needed.
|
||||
- Unlike Ling-3.0-flash (which pairs `--reasoning-parser ling3` / `--tool-call-parser ling3`), Ling-3.0-tiny uses `--reasoning-parser deepseek-r1` and `--tool-call-parser glm45` (its auto-detected template pairing) — the template wraps tool calls in `<tool_call>` blocks and emits an inline `...</think>` chain-of-thought. Toggle them in the **Parsers** card of the [Playground](#playground).
|
||||
- Only `--model-path`, `--host`, and `--port` are needed. SGLang auto-resolves the context length (native 128K from `max_position_embeddings`), the attention backend, and `--mem-fraction-static` from the GPU and the CUDA-graph runtime, so the recipes leave them unset.
|
||||
- The chat template defaults to thinking on. Turn it off per request with `"chat_template_kwargs": {"enable_thinking": false}` for direct answers without the `...</think>` block.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// Measured on lmsysorg/sglang:dev-Ling-3.0-tiny, 1× H200. TTFT/TPOT are P50
|
||||
// (median) from sglang.bench_serving (random ISL 8192 / OSL 1024, --flush-cache);
|
||||
// tokens_per_sec_per_gpu = output tok/s × (isl+osl)/osl. Accuracy from sgl-eval
|
||||
// full GSM8K (1319).
|
||||
// TTFT/TPOT are P50. INT4 uses 80 exact ISL 8192 / OSL 1024 requests with
|
||||
// --flush-cache; BF16/FP8 retain their original published measurements.
|
||||
// Accuracy is full GSM8K (1319).
|
||||
export const benchmarks = [
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "bf16", strategy: "high-throughput", nodes: "single" },
|
||||
@@ -25,4 +24,28 @@ export const benchmarks = [
|
||||
],
|
||||
accuracy: { gsm8k_pct: 94.69 },
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ 8ba213fc",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 90.31, tpot_ms: 1.96, tokens_per_sec_per_gpu: 4398 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 840.04, tpot_ms: 3.55, tokens_per_sec_per_gpu: 32958 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 94.54 },
|
||||
notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 36 shapes through batch 256.",
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
sglang_version: "PR #33561 @ 8ba213fc",
|
||||
speed: [
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 1 },
|
||||
ttft_ms: 305.67, tpot_ms: 6.33, tokens_per_sec_per_gpu: 1359 },
|
||||
{ workload: { dataset: "random", isl: 8192, osl: 1024, max_concurrency: 16 },
|
||||
ttft_ms: 2634.12, tpot_ms: 16.04, tokens_per_sec_per_gpu: 7730 },
|
||||
],
|
||||
accuracy: { gsm8k_pct: 94.54 },
|
||||
notes: "Full GSM8K stop rate 100%; default decode CUDA Graph captured 52 shapes through batch 512. Triton WNA16 used untuned default E=128,N=256 configs.",
|
||||
},
|
||||
];
|
||||
|
||||
@@ -10,6 +10,7 @@ export const config = {
|
||||
quantizations: [
|
||||
{ id: "bf16", label: "BF16" },
|
||||
{ id: "fp8", label: "FP8" },
|
||||
{ id: "int4", label: "INT4" },
|
||||
],
|
||||
strategies: [
|
||||
{ id: "high-throughput", label: "High-Throughput" },
|
||||
@@ -21,6 +22,7 @@ export const config = {
|
||||
modelNames: {
|
||||
"default|bf16": "inclusionAI/Ling-3.0-tiny",
|
||||
"default|fp8": "inclusionAI/Ling-3.0-tiny-fp8",
|
||||
"default|int4": "inclusionAI/Ling-3.0-tiny-int4",
|
||||
},
|
||||
|
||||
placeholders: {
|
||||
@@ -51,13 +53,16 @@ export const config = {
|
||||
--model {{MODEL_NAME}} \\
|
||||
--dataset-name {{DATASET}} \\
|
||||
--random-input-len {{ISL}} --random-output-len {{OSL}} \\
|
||||
--random-range-ratio 1 \\
|
||||
--num-prompts {{NUM_PROMPTS}} --max-concurrency {{MAX_CONCURRENCY}} \\
|
||||
--flush-cache`,
|
||||
accuracy: {
|
||||
gsm8k_pct: `# To install sgl-eval: pip install git+https://github.com/sgl-project/sgl-eval
|
||||
sgl-eval run gsm8k \\
|
||||
--base-url http://{{CURL_HOST}}:{{CURL_PORT}}/v1 \\
|
||||
--num-threads 32`,
|
||||
--num-threads 32 \\
|
||||
--temperature 1.0 --top-p 0.95 \\
|
||||
--thinking`,
|
||||
},
|
||||
},
|
||||
|
||||
@@ -187,5 +192,59 @@ sgl-eval run gsm8k \\
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h20-3e", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h800", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "h100", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "b200", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: true,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
{
|
||||
match: { hw: "gb300", variant: "default", quant: "int4", strategy: "high-throughput", nodes: "single" },
|
||||
verified: false,
|
||||
flags: [
|
||||
"--model-path {{MODEL_NAME}}",
|
||||
"--host {{HOST_IP}}",
|
||||
"--port {{PORT}}",
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user