minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[diffusion] doc: update docs architecture (#27767)

Browse Source
This commit is contained in:
Mick
2026-06-10 14:18:10 +08:00
committed by GitHub
parent f2bcdb0508
commit e8a437ef26
18 changed files with 549 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs_new/custom.css
+120
View File
@@ -83,6 +83,126 @@ html.dark table tbody tr:nth-child(even) td,
background-color: rgba(255,255,255,0.05);
}
.sgd-model-table-wrap {
margin: 1rem 0 1.5rem;
overflow-x: auto;
border: 1px solid rgba(17, 24, 39, 0.12);
border-radius: 8px;
background: rgba(255, 255, 255, 0.55);
scrollbar-width: thin;
}
.sgd-model-table-wrap::-webkit-scrollbar {
width: 8px;
height: 8px;
}
.sgd-model-table {
min-width: 760px;
table-layout: auto;
border-collapse: separate;
border-spacing: 0;
font-size: 0.875rem;
}
.sgd-model-table thead tr {
border-bottom: none;
}
.sgd-model-table thead th {
padding: 11px 14px;
border-bottom: 1px solid rgba(213, 88, 22, 0.24);
background: rgba(213, 88, 22, 0.08) !important;
color: rgb(124, 45, 18);
}
.sgd-model-table tbody td {
padding: 12px 14px;
vertical-align: top;
border-bottom: 1px solid rgba(17, 24, 39, 0.08);
background: transparent !important;
line-height: 1.45;
}
.sgd-model-table tbody tr:last-child td {
border-bottom: none;
}
.sgd-model-table tbody td:first-child {
width: 18%;
font-weight: 650;
color: rgb(31, 41, 55);
}
.sgd-id-list {
display: flex;
flex-wrap: wrap;
gap: 6px;
}
.sgd-id-list code {
display: inline-block;
max-width: 100%;
white-space: normal;
overflow-wrap: anywhere;
font-size: 0.78rem;
line-height: 1.4;
}
.sgd-chip {
display: inline-flex;
align-items: center;
margin: 2px 4px 2px 0;
padding: 2px 8px;
border: 1px solid rgba(213, 88, 22, 0.22);
border-radius: 999px;
background: rgba(213, 88, 22, 0.08);
color: rgb(154, 52, 18);
font-size: 0.75rem;
font-weight: 650;
line-height: 1.45;
}
.sgd-muted {
color: rgb(107, 114, 128);
font-size: 0.85rem;
}
html.dark .sgd-model-table-wrap,
[data-theme="dark"] .sgd-model-table-wrap {
border-color: rgba(255, 255, 255, 0.12);
background: rgba(255, 255, 255, 0.025);
}
html.dark .sgd-model-table thead th,
[data-theme="dark"] .sgd-model-table thead th {
border-bottom-color: rgba(213, 88, 22, 0.32);
background: rgba(213, 88, 22, 0.16) !important;
color: rgb(254, 215, 170);
}
html.dark .sgd-model-table tbody td,
[data-theme="dark"] .sgd-model-table tbody td {
border-bottom-color: rgba(255, 255, 255, 0.08);
}
html.dark .sgd-model-table tbody td:first-child,
[data-theme="dark"] .sgd-model-table tbody td:first-child {
color: rgb(243, 244, 246);
}
html.dark .sgd-chip,
[data-theme="dark"] .sgd-chip {
border-color: rgba(251, 146, 60, 0.3);
background: rgba(251, 146, 60, 0.12);
color: rgb(253, 186, 116);
}
html.dark .sgd-muted,
[data-theme="dark"] .sgd-muted {
color: rgb(156, 163, 175);
}
/* Bold text (**text**) */
.prose strong, .prose b {
font-weight: 600;
docs_new/docs.json
+17 -15
View File
@@ -1204,7 +1204,6 @@
"docs/sglang-diffusion/installation",
"docs/sglang-diffusion/compatibility_matrix",
"docs/sglang-diffusion/disaggregation",
"docs/sglang-diffusion/quantization",
{
"group": "Usage",
"pages": [
@@ -1218,31 +1217,34 @@
"pages": [
"docs/sglang-diffusion/performance-optimization",
"docs/sglang-diffusion/deployment_cookbook",
"docs/sglang-diffusion/ring_sp_performance",
"docs/sglang-diffusion/attention_backends",
"docs/sglang-diffusion/ring_sp_performance",
"docs/sglang-diffusion/dynamic_batching",
{
"group": "Inference Batching",
"group": "Caching Acceleration",
"root": "docs/sglang-diffusion/caching-acceleration",
"tag": "approx",
"pages": [
"docs/sglang-diffusion/dynamic_batching"
"docs/sglang-diffusion/cache_dit",
"docs/sglang-diffusion/teacache"
]
},
"docs/sglang-diffusion/profiling",
"docs/sglang-diffusion/ci_perf"
]
},
{
"group": "Caching Strategies",
"pages": [
"docs/sglang-diffusion/caching-acceleration",
"docs/sglang-diffusion/cache_dit",
"docs/sglang-diffusion/teacache"
"docs/sglang-diffusion/progressive_resolution",
"docs/sglang-diffusion/quantization",
"docs/sglang-diffusion/profiling"
]
},
{
"group": "References",
"pages": [
"docs/sglang-diffusion/environment_variables",
"docs/sglang-diffusion/environment_variables"
]
},
{
"group": "Development",
"pages": [
"docs/sglang-diffusion/support_new_models",
"docs/sglang-diffusion/ci_perf",
"docs/sglang-diffusion/contributing"
]
}
docs_new/docs/sglang-diffusion/api/cli.mdx
+4 -4
View File
@@ -83,7 +83,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis
- `--sp-degree {N}`: sequence parallelism size
- `--ulysses-degree {N}` and `--ring-degree {N}`: USP parallelism controls
- `--enable-cfg-parallel {true|false}`: enable or explicitly disable CFG parallelism
- `--attention-backend {BACKEND}`: attention backend for native SGLang pipelines
- `--attention-backend {BACKEND}`: attention backend for native SGLang and diffusers pipelines
- `--component-attention-backends {MAP}`: per-component attention backend overrides, for example `text_encoder=torch_sdpa,transformer=fa`
- `--attention-backend-config {CONFIG}`: attention backend configuration
@@ -124,7 +124,7 @@ model_path: FastVideo/FastHunyuan-diffusers
prompt: A beautiful woman in a red dress walking down a street
output_path: outputs/
num_gpus: 2
sp_size: 2
sp_degree: 2
tp_size: 1
num_frames: 45
height: 720
@@ -262,7 +262,7 @@ Use `--backend diffusers` to force vanilla diffusers pipelines when no native SG
<td>Choose native SGLang, force native, or force diffusers</td>
</tr>
<tr>
<td><code>--diffusers-attention-backend</code></td>
<td><code>--attention-backend</code></td>
<td><code>flash</code>, <code>_flash_3_hub</code>, <code>sage</code>, <code>xformers</code>, <code>native</code></td>
<td>Attention backend for diffusers pipelines</td>
</tr>
@@ -301,7 +301,7 @@ sglang generate \
--model-path AIDC-AI/Ovis-Image-7B \
--backend diffusers \
--trust-remote-code \
--diffusers-attention-backend flash \
--attention-backend flash \
--prompt "A serene Japanese garden with cherry blossoms" \
--height 1024 \
--width 1024 \
docs_new/docs/sglang-diffusion/attention_backends.mdx
+1
View File
@@ -1,6 +1,7 @@
---
title: "Attention Backends"
description: "Select and configure attention backends for SGLang diffusion pipelines."
tag: "preserve"
---
This document describes the attention backends available in sglang diffusion (`sglang.multimodal_gen`) and how to select them.
docs_new/docs/sglang-diffusion/cache_dit.mdx
+2 -2
View File
@@ -1,6 +1,7 @@
---
title: "Cache-DiT Acceleration"
description: "Configure Cache-DiT acceleration for diffusion inference."
tag: "approx"
---
SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to **1.69x inference speedup** with minimal quality loss.
@@ -559,8 +560,7 @@ SGLang Diffusion x Cache-DiT supports almost all models originally supported in
## Limitations
- **SGLang-native pipelines**: Distributed support (TP/SP) is not yet validated; Cache-DiT will be automatically
disabled when `world_size > 1`.
- **SGLang-native pipelines**: Distributed Cache-DiT paths exist for supported pipelines. Hybrid SP+TP configurations add communication and cache coordination overhead, so validate them on the target model and hardware before using them as production defaults.
- **SCM minimum steps**: SCM requires >= 8 inference steps to be effective
- **Model support**: Only models registered in Cache-DiT's BlockAdapterRegister are supported
docs_new/docs/sglang-diffusion/caching-acceleration.mdx
+5 -3
View File
@@ -1,6 +1,7 @@
---
title: "Caching Acceleration"
description: "Compare caching acceleration strategies for diffusion models."
tag: "approx"
---
SGLang provides two complementary caching strategies for Diffusion Transformer (DiT) models. Both reduce denoising cost by skipping redundant computation, but they operate at different levels.
@@ -70,13 +71,14 @@ See [teacache.md](./teacache) for detailed documentation.
- Tracks L1 distance between modulated inputs across timesteps
- When accumulated distance is below threshold, reuses cached residual
- Supports CFG with separate positive/negative caches
- Uses separate positive/negative caches for supported CFG model families
### Supported Models
- Wan (wan2.1, wan2.2)
- Hunyuan (HunyuanVideo)
- Wan2.1
- Z-Image
- Wan2.2: coefficients are not calibrated yet; enabling TeaCache is accepted but currently no-ops
- HunyuanVideo: not supported yet
For Flux and Qwen models, TeaCache is automatically disabled when CFG is enabled.
docs_new/docs/sglang-diffusion/compatibility_matrix.mdx
+178 -143
View File
@@ -1,20 +1,182 @@
---
title: "Supported Models"
description: "Check model compatibility across diffusion optimizations and backends."
title: "Supported Models and Optimization Compatibility"
description: "Check supported SGLang Diffusion models and their optimization compatibility."
---
The table below shows every supported model and the optimizations supported for them.
This page tracks supported SGLang Diffusion model families and their optimization compatibility. It also covers long-tail models that do not yet have dedicated cookbook recipes.
The symbols used have the following meanings:
For model-specific usage recipes, start from the [Diffusion Cookbook](/cookbook/diffusion/intro). Cookbook pages cover the primary models with examples; this page keeps the compact support and compatibility inventory.
## Supported model inventory
Pass the `Hugging Face Model ID` to `--model-path` for `sglang generate` or `sglang serve`. Python API users can pass the same ID to SGLang Diffusion model-loading helpers.
Missing checkpoint aliases do not imply that a model family is unsupported. The runtime registry may also accept detector-based aliases or local model directories that match the same family.
Rows are grouped when a family shares the same runtime path or optimization support. Use the detailed matrix below when you need per-optimization compatibility.
<Tabs>
<Tab title="Image">
<div className="sgd-model-table-wrap">
<table className="sgd-model-table">
<thead>
<tr>
<th>Model family</th>
<th>Model IDs</th>
</tr>
</thead>
<tbody>
<tr>
<td>FLUX</td>
<td><div className="sgd-id-list"><code>black-forest-labs/FLUX.1-dev</code><code>black-forest-labs/FLUX.2-dev</code><code>black-forest-labs/FLUX.2-dev-NVFP4</code><code>black-forest-labs/FLUX.2-klein-4B</code><code>black-forest-labs/FLUX.2-klein-9B</code><code>black-forest-labs/FLUX.2-klein-base-4B</code><code>black-forest-labs/FLUX.2-klein-base-9B</code></div></td>
</tr>
<tr>
<td>Z-Image</td>
<td><div className="sgd-id-list"><code>Tongyi-MAI/Z-Image</code><code>Tongyi-MAI/Z-Image-Turbo</code></div></td>
</tr>
<tr>
<td>Qwen-Image</td>
<td><div className="sgd-id-list"><code>Qwen/Qwen-Image</code><code>Qwen/Qwen-Image-2512</code><code>Qwen/Qwen-Image-Edit</code><code>Qwen/Qwen-Image-Edit-2509</code><code>Qwen/Qwen-Image-Edit-2511</code><code>Qwen/Qwen-Image-Layered</code></div></td>
</tr>
<tr>
<td>SD3 / SD3.5</td>
<td><div className="sgd-id-list"><code>stabilityai/stable-diffusion-3-medium</code><code>stabilityai/stable-diffusion-3-medium-diffusers</code><code>stabilityai/stable-diffusion-3.5-medium</code><code>stabilityai/stable-diffusion-3.5-medium-diffusers</code><code>stabilityai/stable-diffusion-3.5-large</code><code>stabilityai/stable-diffusion-3.5-large-diffusers</code></div></td>
</tr>
<tr>
<td>SANA</td>
<td><div className="sgd-id-list"><code>Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers</code><code>Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers</code><code>Efficient-Large-Model/Sana_1600M_1024px_diffusers</code><code>Efficient-Large-Model/Sana_600M_1024px_diffusers</code><code>Efficient-Large-Model/Sana_1600M_512px_diffusers</code><code>Efficient-Large-Model/Sana_600M_512px_diffusers</code></div></td>
</tr>
<tr>
<td>FireRed-Image</td>
<td><div className="sgd-id-list"><code>FireRedTeam/FireRed-Image-Edit-1.0</code><code>FireRedTeam/FireRed-Image-Edit-1.1</code></div></td>
</tr>
<tr>
<td>JoyAI-Image</td>
<td><div className="sgd-id-list"><code>jdopensource/JoyAI-Image-Edit-Diffusers</code></div></td>
</tr>
<tr>
<td>Other image pipelines</td>
<td><div className="sgd-id-list"><code>zai-org/GLM-Image</code><code>tencent/Hunyuan3D-2</code><code>baidu/ERNIE-Image</code><code>baidu/ERNIE-Image-Turbo</code><code>ideogram-ai/ideogram-4-fp8</code><code>ideogram-ai/ideogram-4-nf4</code><code>Comfy-Org/Ideogram-4</code></div></td>
</tr>
</tbody>
</table>
</div>
</Tab>
<Tab title="Video">
<div className="sgd-model-table-wrap">
<table className="sgd-model-table">
<thead>
<tr>
<th>Model family</th>
<th>Model IDs</th>
<th>Resolution / mode</th>
<th>Optimization support</th>
</tr>
</thead>
<tbody>
<tr>
<td>FastWan</td>
<td><div className="sgd-id-list"><code>FastVideo/FastWan2.1-T2V-1.3B-Diffusers</code><code>FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers</code><code>FastVideo/FastWan2.2-TI2V-5B-Diffusers</code></div></td>
<td>480p / 720p</td>
<td><span className="sgd-chip">VSA</span></td>
</tr>
<tr>
<td>Wan2.2</td>
<td><div className="sgd-id-list"><code>Wan-AI/Wan2.2-TI2V-5B-Diffusers</code><code>Wan-AI/Wan2.2-T2V-A14B-Diffusers</code><code>nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4</code><code>Wan-AI/Wan2.2-I2V-A14B-Diffusers</code></div></td>
<td>TI2V / T2V / I2V, 480p / 720p</td>
<td><span className="sgd-chip">Sage</span><span className="sgd-chip">Laser</span><span className="sgd-chip">BSA</span><span className="sgd-chip">Rain Fusion</span></td>
</tr>
<tr>
<td>HunyuanVideo</td>
<td><div className="sgd-id-list"><code>hunyuanvideo-community/HunyuanVideo</code><code>FastVideo/FastHunyuan-diffusers</code></div></td>
<td>720×1280 / 544×960</td>
<td><span className="sgd-chip">Tile</span><span className="sgd-chip">Sage</span><span className="sgd-chip">SVG2</span></td>
</tr>
<tr>
<td>Wan2.1</td>
<td><div className="sgd-id-list"><code>Wan-AI/Wan2.1-T2V-1.3B-Diffusers</code><code>Wan-AI/Wan2.1-T2V-14B-Diffusers</code><code>Wan-AI/Wan2.1-I2V-14B-480P-Diffusers</code><code>Wan-AI/Wan2.1-I2V-14B-720P-Diffusers</code></div></td>
<td>T2V / I2V, 480p / 720p</td>
<td><span className="sgd-chip">TeaCache</span><span className="sgd-chip">Tile</span><span className="sgd-chip">Sage</span><span className="sgd-chip">SVG2</span><span className="sgd-chip">Laser</span><span className="sgd-chip">BSA</span><span className="sgd-chip">Rain Fusion</span></td>
</tr>
<tr>
<td>TurboWan</td>
<td><div className="sgd-id-list"><code>IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers</code><code>IPostYellow/TurboWan2.1-T2V-14B-Diffusers</code><code>IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers</code><code>IPostYellow/TurboWan2.2-I2V-A14B-Diffusers</code></div></td>
<td>480p / 720p</td>
<td><span className="sgd-chip">TeaCache</span><span className="sgd-chip">SLA</span><span className="sgd-chip">SageSLA</span></td>
</tr>
<tr>
<td>MOVA</td>
<td><div className="sgd-id-list"><code>OpenMOSS-Team/MOVA-360p</code><code>OpenMOSS-Team/MOVA-720p</code></div></td>
<td>Video-audio, 360p / 720p; local MOVA detector aliases are also supported.</td>
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
</tr>
<tr>
<td>Wan2.1 Fun</td>
<td><div className="sgd-id-list"><code>weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers</code></div></td>
<td>480p inpainting</td>
<td><span className="sgd-chip">TeaCache</span><span className="sgd-chip">Tile</span><span className="sgd-chip">Sage</span><span className="sgd-chip">SVG2</span></td>
</tr>
<tr>
<td>Helios</td>
<td><div className="sgd-id-list"><code>BestWishYsh/Helios-Base</code><code>BestWishYsh/Helios-Mid</code><code>BestWishYsh/Helios-Distilled</code></div></td>
<td>720p</td>
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
</tr>
<tr>
<td>LTX-2</td>
<td><div className="sgd-id-list"><code>Lightricks/LTX-2</code><code>Lightricks/LTX-2.3</code></div></td>
<td>One-stage, two-stage, TI2V, HQ</td>
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
</tr>
<tr>
<td>Cosmos3</td>
<td><div className="sgd-id-list"><code>nvidia/Cosmos3-Nano</code><code>nvidia/Cosmos3-Super</code><code>nvidia/Cosmos3-Super-Text2Image</code><code>nvidia/Cosmos3-Super-Image2Video</code></div></td>
<td>T2V / I2V / T2I</td>
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
</tr>
</tbody>
</table>
</div>
</Tab>
<Tab title="Realtime / World">
<div className="sgd-model-table-wrap">
<table className="sgd-model-table">
<thead>
<tr>
<th>Model family</th>
<th>Model IDs / detector</th>
<th>Notes</th>
</tr>
</thead>
<tbody>
<tr>
<td>LingBotWorld</td>
<td><div className="sgd-id-list"><code>robbyant/lingbot-world-fast-diffusers</code></div></td>
<td>Realtime world model with causal state and control tokens.</td>
</tr>
<tr>
<td>SANA-WM</td>
<td><div className="sgd-id-list"><code>Efficient-Large-Model/SANA-WM_bidirectional</code><code>Efficient-Large-Model/SANA-WM_streaming</code></div></td>
<td>World-model pipeline with bidirectional and streaming checkpoints.</td>
</tr>
</tbody>
</table>
</div>
</Tab>
</Tabs>
<Note>
Wan2.2 TI2V 5B currently has known quality issues when used for I2V generation.
</Note>
## Optimization compatibility
The detailed video matrix uses these symbols:
- ✅ = Full compatibility
- ❌ = No compatibility
- ⭕ = Does not apply to this model
## Models x Optimization
The `HuggingFace Model ID` can be passed directly to `from_pretrained()` methods, and sglang-diffusion will use the
optimal
default parameters when initializing and generating videos.
<Accordion title="Detailed video optimization matrix">
### Video Generation Models
@@ -81,8 +243,8 @@ Optimization columns are abbreviated to keep the matrix readable:
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FastWan2.2 TI2V 5B Full Attn</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers`</td>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FastWan2.2 TI2V 5B</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers</code><br /><code>FastVideo/FastWan2.2-TI2V-5B-Diffusers</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>720p</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>⭕</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>⭕</td>
@@ -112,7 +274,7 @@ Optimization columns are abbreviated to keep the matrix readable:
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Wan2.2 T2V A14B</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`Wan-AI/Wan2.2-T2V-A14B-Diffusers`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Wan-AI/Wan2.2-T2V-A14B-Diffusers</code><br /><code>nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4</code></td>
<td style={{padding: "9px 8px", backgroundColor: "rgba(255,255,255,0.02)"}}>480p<br />720p</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
@@ -397,7 +559,7 @@ Optimization columns are abbreviated to keep the matrix readable:
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Cosmos3-Super (T2V / I2V / T2I)</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>nvidia/Cosmos3-Super</code></td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>nvidia/Cosmos3-Super</code><br /><code>nvidia/Cosmos3-Super-Text2Image</code><br /><code>nvidia/Cosmos3-Super-Image2Video</code></td>
<td style={{padding: "9px 8px", backgroundColor: "rgba(255,255,255,0.02)"}}>720p · 480p<br />1024×1024 (T2I)</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
@@ -435,135 +597,7 @@ Optimization columns are abbreviated to keep the matrix readable:
`transformer/config.json` at load time. A single checkpoint serves T2V,
I2V (`--image-path`), and T2I (`--num-frames 1`).
### Image Generation Models
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "22%"}} />
<col style={{width: "46%"}} />
<col style={{width: "32%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Model Name</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>HuggingFace Model ID</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FLUX.1-dev</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`black-forest-labs/FLUX.1-dev`</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FLUX.2-dev</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`black-forest-labs/FLUX.2-dev`</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FLUX.2-dev-NVFP4</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>black-forest-labs/FLUX.2-dev-NVFP4</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FLUX.2-Klein-4B</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>black-forest-labs/FLUX.2-klein-4B</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FLUX.2-Klein-9B</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>black-forest-labs/FLUX.2-klein-9B</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Z-Image</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Tongyi-MAI/Z-Image</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Z-Image-Turbo</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Tongyi-MAI/Z-Image-Turbo</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>GLM-Image</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>zai-org/GLM-Image</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen Image</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Qwen/Qwen-Image</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen Image 2512</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Qwen/Qwen-Image-2512</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen Image Edit</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`Qwen/Qwen-Image-Edit`</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen Image Edit 2509</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Qwen/Qwen-Image-Edit-2509</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen Image Edit 2511</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Qwen/Qwen-Image-Edit-2511</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Qwen Image Layered</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Qwen/Qwen-Image-Layered</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SD3 Medium</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>stabilityai/stable-diffusion-3-medium-diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SD3.5 Medium</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>stabilityai/stable-diffusion-3.5-medium-diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SD3.5 Large</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>stabilityai/stable-diffusion-3.5-large-diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Hunyuan3D-2</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>tencent/Hunyuan3D-2</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SANA 1.5 1.6B</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SANA 1.5 4.8B</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SANA 1600M 1024px</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Efficient-Large-Model/Sana_1600M_1024px_diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SANA 600M 1024px</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Efficient-Large-Model/Sana_600M_1024px_diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SANA 1600M 512px</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Efficient-Large-Model/Sana_1600M_512px_diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>SANA 600M 512px</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>Efficient-Large-Model/Sana_600M_512px_diffusers</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FireRed-Image-Edit 1.0</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>FireRedTeam/FireRed-Image-Edit-1.0</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FireRed-Image-Edit 1.1</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>FireRedTeam/FireRed-Image-Edit-1.1</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>ERNIE-Image</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>baidu/ERNIE-Image</code></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>ERNIE-Image-Turbo</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>baidu/ERNIE-Image-Turbo</code></td>
</tr>
</tbody>
</table>
</Accordion>
## Supported Components
@@ -599,7 +633,8 @@ Use the component name from the pipeline's `model_index.json` or the native pipe
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "20%"}} />
<col style={{width: "80%"}} />
<col style={{width: "42%"}} />
<col style={{width: "38%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
docs_new/docs/sglang-diffusion/deployment_cookbook.mdx
+4 -3
View File
@@ -1,9 +1,10 @@
---
title: "Deployment Cookbook"
description: "Practical defaults for choosing CPU offload, FSDP, CFG parallelism, SP, and TP in SGLang Diffusion."
title: "Deployment and Performance Modes"
description: "Choose CPU offload, FSDP, CFG parallelism, SP, TP, and performance-mode presets in SGLang Diffusion."
tag: "preserve"
---
This page gives practical defaults for choosing CPU offload, FSDP, CFG parallelism, SP, and TP.
This page gives practical defaults for choosing `--performance-mode`, CPU offload, FSDP, CFG parallelism, SP, and TP.
## Quick Rule
docs_new/docs/sglang-diffusion/disaggregation.mdx
+5 -1
View File
@@ -269,9 +269,13 @@ Set `--disagg-p2p-hostname` to the actual IP on each machine. For multi-machine,
<td><code>--denoiser-tp</code> / <code>--denoiser-sp</code> / <code>--denoiser-ulysses</code> / <code>--denoiser-ring</code></td>
<td>Denoiser parallelism</td>
</tr>
<tr>
<td><code>--decoder-sp</code></td>
<td>Decoder sequence parallelism</td>
</tr>
<tr>
<td><code>--decoder-tp</code></td>
<td>Decoder tensor parallelism</td>
<td>Deprecated alias for <code>--decoder-sp</code></td>
</tr>
</tbody>
</table>
docs_new/docs/sglang-diffusion/dynamic_batching.mdx
+2 -1
View File
@@ -1,6 +1,7 @@
---
title: "Inference Batching"
description: "Batch compatible native SGLang-Diffusion requests during serving."
tag: "preserve"
mode: wide
---
Dynamic batching is an opt-in SGLang-Diffusion serving mode that merges compatible queued requests into one native pipeline batch. It is separate from LLM continuous batching and tokenizer batching.
@@ -40,7 +41,7 @@ Use `--batching-config /path/to/batching_config.json` to load JSON rules when a
## Compatibility
An initial implementation of dynamic batching for T2I and T2V models can be found in [#18764](https://github.com/sgl-project/sglang/pull/18764). The current compatibility grid is below and will be updated as more coverage is added. See [Supported Models](./compatibility_matrix) for full model IDs.
An initial implementation of dynamic batching for T2I and T2V models can be found in [#18764](https://github.com/sgl-project/sglang/pull/18764). The current compatibility grid is below and will be updated as more coverage is added. See [Supported Models and Optimization Compatibility](./compatibility_matrix) for common model IDs and optimization support.
`✅` means supported, `❌` means not currently supported, `?` means untested, and `-` means not applicable.
docs_new/docs/sglang-diffusion/index.mdx
+11 -5
View File
@@ -30,11 +30,10 @@ sglang serve --model-path Qwen/Qwen-Image --port 30010
## Start Here
- [Installation](/docs/sglang-diffusion/installation): install SGLang Diffusion and platform dependencies
- [Compatibility Matrix](/docs/sglang-diffusion/compatibility_matrix): check model, optimization, and component override support
- [Supported Models and Optimization Compatibility](/docs/sglang-diffusion/compatibility_matrix): check supported model families, long-tail coverage, and optimization support
- [CLI](/docs/sglang-diffusion/api/cli): run one-off generation jobs or launch a persistent server
- [OpenAI-Compatible API](/docs/sglang-diffusion/api/openai_api): send image and video requests to the HTTP server
- [Attention Backends](/docs/sglang-diffusion/attention_backends): choose the best backend for your model and hardware
- [Inference Batching](/docs/sglang-diffusion/dynamic_batching): batch compatible native diffusion requests during serving
- [Performance Overview](/docs/sglang-diffusion/performance-optimization): choose speed, memory, parallelism, caching, and quality-tradeoff levers
- [Caching Acceleration](/docs/sglang-diffusion/caching-acceleration): use Cache-DiT or TeaCache to reduce denoising cost
- [Quantization](/docs/sglang-diffusion/quantization): load quantized transformer checkpoints
- [Contributing](/docs/sglang-diffusion/contributing): contribution workflow, adding new models, and CI perf baselines
@@ -42,10 +41,17 @@ sglang serve --model-path Qwen/Qwen-Image --port 30010
## Additional Documentation
- [Post-Processing](/docs/sglang-diffusion/api/post_processing): frame interpolation and upscaling
- [Performance Overview](/docs/sglang-diffusion/performance-optimization): overview of attention, caching, and profiling
- [Deployment and Performance Modes](/docs/sglang-diffusion/deployment_cookbook): choose `--performance-mode`, offload, FSDP, CFG parallelism, SP, and TP
- [Attention Backends](/docs/sglang-diffusion/attention_backends): choose the best backend for your model and hardware
- [Sequence Parallelism](/docs/sglang-diffusion/ring_sp_performance): configure SP, Ulysses, and ring-based splitting for long sequences
- [Inference Batching](/docs/sglang-diffusion/dynamic_batching): batch compatible native diffusion requests during serving
- [Progressive Resolution Generation](/docs/sglang-diffusion/progressive_resolution): run early denoising steps at lower latent resolution for selected pipelines
- [Environment Variables](/docs/sglang-diffusion/environment_variables): platform, caching, storage, and debugging configuration
## Developer Documentation
- [Support New Models](/docs/sglang-diffusion/support_new_models): implementation guide for new diffusion pipelines
- [CI Performance](/docs/sglang-diffusion/ci_perf): performance baseline generation
- [CI Performance Baselines](/docs/sglang-diffusion/ci_perf): generate and update performance baselines used in CI
## References
docs_new/docs/sglang-diffusion/installation.mdx
+7 -5
View File
@@ -57,7 +57,7 @@ docker run --gpus all \
## Platform-Specific: ROCm (AMD GPUs)
For AMD Instinct GPUs (e.g., MI300X), you can use the ROCm-enabled Docker image:
For AMD Instinct GPUs (e.g., MI300X), use a ROCm-enabled Docker image from [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang). The tag below is an example for ROCm 7.0 on MI300X and may lag the latest release tag:
```bash Command
docker run --device=/dev/kfd --device=/dev/dri --ipc=host \
@@ -71,7 +71,7 @@ For detailed ROCm system configuration and installation from source, see [AMD GP
## Platform-Specific: MUSA (Moore Threads GPUs)
For Moore Threads GPUs (MTGPU) with the MUSA software stack, please follow the instructions below to install from source:
For Moore Threads GPUs (MTGPU) with the MUSA software stack, follow the platform guide first. If the source tree still requires the alternate platform `pyproject` fallback, keep a backup of the default file before switching:
```bash Command
# Clone the repository
@@ -80,7 +80,8 @@ cd sglang
# Install the Python packages
pip install --upgrade pip
rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml
mv python/pyproject.toml python/pyproject.toml.bak
cp python/pyproject_other.toml python/pyproject.toml
pip install -e "python[all_musa]"
```
@@ -106,7 +107,7 @@ sglang generate --model-path black-forest-labs/FLUX.1-dev \
## Platform-Specific: Apple MPS
For Apple MPS, please follow the instructions below to install from source:
For Apple MPS, follow the instructions below to install from source. If the source tree still requires the alternate platform `pyproject` fallback, keep a backup of the default file before switching:
```bash Command
# Install ffmpeg
@@ -125,6 +126,7 @@ source sglang-diffusion/bin/activate
# Install the Python packages
uv pip install --upgrade pip
rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml
mv python/pyproject.toml python/pyproject.toml.bak
cp python/pyproject_other.toml python/pyproject.toml
uv pip install -e "python[all_mps]"
```
docs_new/docs/sglang-diffusion/performance-optimization.mdx
+91 -38
View File
@@ -1,73 +1,126 @@
---
title: "Performance Optimization"
description: "Optimize SGLang diffusion performance with caching, kernels, and profiling."
description: "Choose performance levers for SGLang Diffusion by latency, throughput, memory, and quality tradeoffs."
---
This section covers the main performance levers for SGLang Diffusion: attention backends, caching acceleration, and profiling.
## Overview
Use this page as the starting point for SGLang Diffusion performance work. It separates performance levers into two decision classes:
- **Output-preserving / lossless-style:** system settings that should preserve model behavior while changing residency, parallelism, kernels, or scheduling.
- **Quality-tradeoff / lossy or approximate:** techniques that can change the denoising path, numerical representation, or generated output.
The docs use "output-preserving" instead of promising bit-exact "lossless" because different kernels, GPU types, or precision paths can still introduce small numerical differences. The decision boundary is whether the optimization intentionally trades quality or output equivalence for speed.
## Start Here
1. Pick a serving or generation mode from [Deployment and Performance Modes](./deployment_cookbook). `--performance-mode auto` is the default; use `speed` when the model fits in GPU memory and latency matters most, `memory` when GPU memory is the bottleneck, and `manual` when every performance flag should be explicit.
2. Choose the right attention backend from [Attention Backends](./attention_backends).
3. Use [Sequence Parallelism](./ring_sp_performance) only when the model and video shape benefit from sequence splitting.
4. Use [Inference Batching](./dynamic_batching) for concurrent compatible requests during serving.
5. Use [Profiling](./profiling) before changing several levers at once.
## Output-Preserving / Lossless-Style Levers
These settings should preserve model behavior while changing residency, parallelism, kernels, or scheduling. They are the first choices for production tuning.
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "22%"}} />
<col style={{width: "18%"}} />
<col style={{width: "60%"}} />
<col style={{width: "24%"}} />
<col style={{width: "30%"}} />
<col style={{width: "46%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Optimization</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.05)"}}>Type</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700, whiteSpace: "nowrap", backgroundColor: "rgba(255,255,255,0.02)"}}>Description</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Lever</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Use when</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Docs</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Cache-DiT</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Caching</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Block-level caching with DBCache, TaylorSeer, and SCM</td>
<td style={{padding: "9px 12px", fontWeight: 500}}><code>--performance-mode</code></td>
<td style={{padding: "9px 12px"}}>You want a safe preset for speed or memory without overriding explicit flags.</td>
<td style={{padding: "9px 12px"}}><a href="./deployment_cookbook">Deployment and Performance Modes</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>TeaCache</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Caching</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Timestep-level caching based on temporal similarity</td>
<td style={{padding: "9px 12px", fontWeight: 500}}>Offload, FSDP, CFG parallelism</td>
<td style={{padding: "9px 12px"}}>GPU memory, multi-GPU residency, or CFG branch splitting is the main bottleneck.</td>
<td style={{padding: "9px 12px"}}><a href="./deployment_cookbook">Deployment and Performance Modes</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Attention Backends</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Kernel</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Optimized attention implementations (FlashAttention, SageAttention, etc.)</td>
<td style={{padding: "9px 12px", fontWeight: 500}}>Sequence parallelism</td>
<td style={{padding: "9px 12px"}}>Long image/video sequences need sequence-level parallelism.</td>
<td style={{padding: "9px 12px"}}><a href="./ring_sp_performance">Sequence Parallelism</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Inference Batching</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Scheduler</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Request batching for native diffusion serving</td>
<td style={{padding: "9px 12px", fontWeight: 500}}>Attention backend</td>
<td style={{padding: "9px 12px"}}>Kernel choice dominates DiT latency or memory.</td>
<td style={{padding: "9px 12px"}}><a href="./attention_backends">Attention Backends</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Profiling</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Diagnostics</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>PyTorch Profiler and Nsight Systems guidance</td>
<td style={{padding: "9px 12px", fontWeight: 500}}>Dynamic batching</td>
<td style={{padding: "9px 12px"}}>Serving many compatible requests concurrently.</td>
<td style={{padding: "9px 12px"}}><a href="./dynamic_batching">Inference Batching</a></td>
</tr>
</tbody>
</table>
## Start Here
## Quality-Tradeoff / Lossy Or Approximate Levers
- Use [Attention Backends](./attention_backends) to choose the best backend for your model and hardware.
- Use [Inference Batching](./dynamic_batching) to improve throughput for compatible concurrent requests.
- Use [Caching Acceleration](./caching-acceleration) to reduce denoising cost with Cache-DiT or TeaCache.
- Use [Profiling](./profiling) when you need to diagnose a bottleneck rather than guess.
These techniques can change the denoising path, numerical representation, or generated output. They are useful after you have a baseline and an acceptance criterion for quality.
## Caching at a Glance
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "24%"}} />
<col style={{width: "30%"}} />
<col style={{width: "46%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Lever</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Tradeoff</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Docs</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500}}>Cache-DiT</td>
<td style={{padding: "9px 12px"}}>Skips selected DiT block or step computation based on cache decisions.</td>
<td style={{padding: "9px 12px"}}><a href="./cache_dit">Cache-DiT</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500}}>TeaCache</td>
<td style={{padding: "9px 12px"}}>Reuses residuals when consecutive denoising steps are similar enough.</td>
<td style={{padding: "9px 12px"}}><a href="./teacache">TeaCache</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500}}>Progressive resolution</td>
<td style={{padding: "9px 12px"}}>Runs early denoising at lower latent resolution for supported pipelines.</td>
<td style={{padding: "9px 12px"}}><a href="./progressive_resolution">Progressive Resolution Generation</a></td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500}}>Quantization</td>
<td style={{padding: "9px 12px"}}>Uses lower-precision transformer weights or activations.</td>
<td style={{padding: "9px 12px"}}><a href="./quantization">Quantization</a></td>
</tr>
</tbody>
</table>
- [Cache-DiT](./cache_dit) is block-level caching for diffusers pipelines and higher speedup-oriented tuning.
- [TeaCache](./teacache) is timestep-level caching built into SGLang model families.
## Practical Order
1. Establish a baseline with the target model, resolution, frame count, step count, and GPU type.
2. Select `--performance-mode` and explicit residency or parallelism flags.
3. Tune attention backend and batching for the deployment pattern.
4. Profile if the bottleneck is unclear.
5. Add caching, progressive resolution, or quantization only after comparing output quality against your acceptance target.
## Current Baseline Snapshot
## Diagnostics
For Ring SP benchmark details, see:
- [Ring SP Performance](./ring_sp_performance)
[Profiling](./profiling) is not an optimization technique by itself. It belongs in the performance workflow because it tells you which stage, kernel, or denoising step is worth optimizing before you change multiple levers.
## References
- [Cache-DiT Repository](https://github.com/vipshop/cache-dit)
- [TeaCache Paper](https://arxiv.org/abs/2411.14324)
- [Deployment and Performance Modes](./deployment_cookbook)
- [Attention Backends](./attention_backends)
- [Sequence Parallelism](./ring_sp_performance)
- [Caching Strategies](./caching-acceleration)
- [Profiling](./profiling)
docs_new/docs/sglang-diffusion/profiling.mdx
+1
View File
@@ -1,6 +1,7 @@
---
title: "Profiling"
description: "Profile SGLang diffusion workloads with PyTorch Profiler and Nsight Systems."
tag: "diag"
---
This guide covers profiling techniques for multimodal generation pipelines in SGLang.
docs_new/docs/sglang-diffusion/progressive_resolution.mdx
+2 -3
View File
@@ -1,19 +1,18 @@
---
title: "Progressive Resolution Generation"
description: "Experimental spectral progressive resolution growing for selected SGLang Diffusion pipelines."
tag: "approx"
---
Progressive resolution growing is an experimental feature for selected SGLang Diffusion pipelines. It runs early denoising steps at a coarser latent resolution and spectrally upsamples the latent before the full-resolution steps. On the benchmark setup below, this reduces the quadratic attention cost of the DiT transformer and yields up to **1.63× speedup on FLUX.1**, **1.93× speedup on FLUX.2**, **2.33× speedup on Z-Image**, **2.78× speedup on Wan 2.1 T2V**, and **1.69× speedup on Qwen-Image**.
This page is intentionally not linked from the main documentation navigation while the feature is still experimental.
Based on [Spectral Progressive Diffusion (arXiv 2605.18736)](https://arxiv.org/abs/2605.18736).
## Overview
DiT attention is O(n²) in sequence length. Running the first N denoising steps at half the spatial resolution cuts the attention cost to ~6% for those steps.
The transition point — how many steps to run at each resolution — is computed from the **Bayes-optimal frequency-activation criterion**: frequencies that cannot be resolved at the coarse scale are not denoised there. This makes the speedup lossless by construction.
The transition point — how many steps to run at each resolution — is computed from the **Bayes-optimal frequency-activation criterion**: frequencies that cannot be resolved at the coarse scale are not denoised there. The method is designed to preserve quality under this criterion, but generated outputs can still differ from the full-resolution baseline.
| Model | Full-res tokens | Half-res tokens | Token-step ratio |
|-------|----------------|----------------|-----------------|
docs_new/docs/sglang-diffusion/quantization.mdx
+1
View File
@@ -1,5 +1,6 @@
---
title: "Quantization"
tag: "approx"
metatags:
description: "SGLang-Diffusion supports quantized transformer checkpoints. In most cases, keep the base model and the quantized transformer override separate."
---
docs_new/docs/sglang-diffusion/ring_sp_performance.mdx
+83 -26
View File
@@ -1,46 +1,108 @@
---
title: "Ring SP Benchmark: Wan2.2-TI2V-5B (u1r2 vs Baseline)"
title: "Sequence Parallelism"
tag: "preserve"
metatags:
description: "Review Ring-SP benchmark results for Wan2.2-TI2V-5B-Diffusers in SGLang Diffusion."
description: "Configure sequence parallelism, Ulysses, and ring-based sequence splitting for SGLang Diffusion workloads."
---
This page reports Ring-SP performance for `Wan2.2-TI2V-5B-Diffusers` using:
Sequence parallelism splits long image or video latent sequences across GPUs. In SGLang Diffusion, the public controls are:
- Parallel config: `sp=2, ulysses=1, ring=2` (short: `u1r2`)
- Baseline config: `sp=1, ulysses=1, ring=1` (short: `u1r1`)
- `--sp-degree`: total sequence parallel degree
- `--ulysses-degree`: Ulysses parallel degree
- `--ring-degree`: ring parallel degree
## Benchmark Setup
The degrees must satisfy:
- Model: `Wan2.2-TI2V-5B-Diffusers`
- GPU: `48G RTX40 series * 2`
```text
sp_degree = ulysses_degree * ring_degree
```
## Online Serving
Use SP when sequence length or video shape makes the DiT forward pass the bottleneck and the model supports sequence sharding. For latency-oriented multi-GPU Qwen/Wan deployments, also compare against CFG parallelism and FSDP; SP is not automatically the best multi-GPU setting for every model.
### Ring SP (`u1r2`)
## Recommended Commands
### Two-GPU Sequence Parallelism
This example uses two GPUs with `sp=2`, `ulysses=1`, and `ring=2`.
```bash
sglang serve \
--model-type diffusion \
--model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \
--num-gpus 2 --sp-degree 2 --ulysses-degree 1 --ring-degree 2 \
--model-path Wan-AI/Wan2.2-TI2V-5B-Diffusers \
--num-gpus 2 \
--sp-degree 2 \
--ulysses-degree 1 \
--ring-degree 2 \
--port 8898
```
### Baseline (`u1r1`)
### Single-GPU Baseline
Use an explicit single-GPU baseline before attributing a gain to sequence parallelism.
```bash
sglang serve \
--model-type diffusion \
--model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \
--num-gpus 1 --sp-degree 1 --ulysses-degree 1 --ring-degree 1 \
--model-path Wan-AI/Wan2.2-TI2V-5B-Diffusers \
--num-gpus 1 \
--sp-degree 1 \
--ulysses-degree 1 \
--ring-degree 1 \
--port 8898
```
## Benchmarks
## Choosing The Degrees
### Benchmark Disclaimer
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
<col style={{width: "28%"}} />
<col style={{width: "32%"}} />
<col style={{width: "40%"}} />
</colgroup>
<thead>
<tr style={{borderBottom: "2px solid #d55816"}}>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Setting</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Typical use</th>
<th style={{textAlign: "left", padding: "10px 12px", fontWeight: 700}}>Notes</th>
</tr>
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px"}}><code>--sp-degree 1</code></td>
<td style={{padding: "9px 12px"}}>Single-GPU or no sequence splitting</td>
<td style={{padding: "9px 12px"}}>Use this as the baseline.</td>
</tr>
<tr>
<td style={{padding: "9px 12px"}}><code>--ulysses-degree N</code></td>
<td style={{padding: "9px 12px"}}>Ulysses-only sequence parallelism</td>
<td style={{padding: "9px 12px"}}>When ring parallelism is not needed, keep <code>--ring-degree 1</code>.</td>
</tr>
<tr>
<td style={{padding: "9px 12px"}}><code>--ring-degree N</code></td>
<td style={{padding: "9px 12px"}}>Ring-based sequence splitting over long sequences</td>
<td style={{padding: "9px 12px"}}>Keep <code>--sp-degree</code> equal to <code>ulysses_degree * ring_degree</code>.</td>
</tr>
</tbody>
</table>
These benchmarks are provided for reference under one specific setup and command configuration. Actual performance may vary with model settings, runtime environment, and request patterns.
## Benchmarking Guidance
When benchmarking SP, compare the same model, precision, resolution, frame count, step count, scheduler settings, prompt type, and output path. Report both stage latency and peak GPU memory; SP can reduce per-GPU memory while adding communication overhead.
Useful metrics:
- End-to-end latency
- Denoising stage latency
- Decoding stage latency
- Peak GPU memory and peak allocated memory
- Communication or runtime overhead when available
## Reference Benchmark
The following numbers are a reference measurement for one setup. They are not a general promise for all Wan2.2 deployments.
- Model: `Wan-AI/Wan2.2-TI2V-5B-Diffusers`
- Hardware: two 48 GB RTX 40-series GPUs for sequence parallelism, one 48 GB RTX 40-series GPU for baseline
- Sequence parallel config: `sp=2, ulysses=1, ring=2` (`u1r2`)
- Baseline config: `sp=1, ulysses=1, ring=1` (`u1r1`)
### Stage Time Breakdown
@@ -150,9 +212,4 @@ These benchmarks are provided for reference under one specific setup and command
</tbody>
</table>
## Summary
- End-to-end latency improves from `90.63s` to `63.74s` (`1.42x`).
- Main gains come from `Denoising` (`1.36x`) and `Decoding` (`1.75x`).
- Absolute memory usage drops noticeably on Ring-SP (`Peak GPU Memory -7.33GB`, `Peak Allocated -7.05GB`).
- Overhead ratio rises (`+7.9pp`), so future tuning can focus on reducing communication/runtime overhead while preserving the latency gain.
In this setup, end-to-end latency improved from `90.63s` to `63.74s` (`1.42x`) and peak GPU memory dropped by `7.33GB`. The overhead ratio increased, so future tuning should still check communication and runtime overhead on the target hardware.
docs_new/docs/sglang-diffusion/teacache.mdx
+15 -9
View File
@@ -1,6 +1,7 @@
---
title: "TeaCache Acceleration"
description: "Configure TeaCache for temporal similarity-based diffusion acceleration."
tag: "approx"
---
> **Note**: This is one of two caching strategies available in SGLang.
@@ -14,7 +15,7 @@ TeaCache works by:
1. Tracking the L1 distance between modulated inputs across consecutive timesteps
2. Accumulating the rescaled L1 distance over steps
3. When accumulated distance is below a threshold, reusing the cached residual
4. Supporting CFG (Classifier-Free Guidance) with separate positive/negative caches
4. Using separate positive/negative caches for supported CFG model families
## How It Works
@@ -39,11 +40,11 @@ accumulated += poly(coefficients)(rel_l1)
### CFG Support
For models that support CFG cache separation (Wan, Hunyuan, Z-Image), TeaCache maintains separate caches for positive and negative branches:
For models that support CFG cache separation, TeaCache maintains separate caches for positive and negative branches:
- `previous_modulated_input` / `previous_residual` for positive branch
- `previous_modulated_input_negative` / `previous_residual_negative` for negative branch
For models that don't support CFG separation (Flux, Qwen), TeaCache is automatically disabled when CFG is enabled.
For models that do not support CFG separation, TeaCache is automatically disabled when CFG is enabled.
## Configuration
@@ -77,7 +78,7 @@ params = TeaCacheParams(
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`teacache_thresh`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>float</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Threshold for accumulated L1 distance. Lower = more caching, faster but potentially lower quality</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Threshold for accumulated L1 distance. Higher = more caching, faster but potentially lower quality</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`coefficients`</td>
@@ -93,7 +94,7 @@ Different models may have different optimal configurations. The coefficients are
## Supported Models
TeaCache is built into the following model families:
TeaCache support status by model family:
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
<colgroup>
@@ -110,19 +111,24 @@ TeaCache is built into the following model families:
</thead>
<tbody>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Wan (wan2.1, wan2.2)</td>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Wan2.1</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Yes</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Full support</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Hunyuan (HunyuanVideo)</td>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Wan2.2</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Yes</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>To be supported</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Coefficients are not calibrated yet; enabling TeaCache is accepted but currently no-ops</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Z-Image</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Yes</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>To be supported</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Full support</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>HunyuanVideo</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>No</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Not supported yet</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>Flux</td>