diff --git a/docs_new/custom.css b/docs_new/custom.css index 3dd898727..e1fd33ba2 100644 --- a/docs_new/custom.css +++ b/docs_new/custom.css @@ -83,6 +83,126 @@ html.dark table tbody tr:nth-child(even) td, background-color: rgba(255,255,255,0.05); } +.sgd-model-table-wrap { + margin: 1rem 0 1.5rem; + overflow-x: auto; + border: 1px solid rgba(17, 24, 39, 0.12); + border-radius: 8px; + background: rgba(255, 255, 255, 0.55); + scrollbar-width: thin; +} + +.sgd-model-table-wrap::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +.sgd-model-table { + min-width: 760px; + table-layout: auto; + border-collapse: separate; + border-spacing: 0; + font-size: 0.875rem; +} + +.sgd-model-table thead tr { + border-bottom: none; +} + +.sgd-model-table thead th { + padding: 11px 14px; + border-bottom: 1px solid rgba(213, 88, 22, 0.24); + background: rgba(213, 88, 22, 0.08) !important; + color: rgb(124, 45, 18); +} + +.sgd-model-table tbody td { + padding: 12px 14px; + vertical-align: top; + border-bottom: 1px solid rgba(17, 24, 39, 0.08); + background: transparent !important; + line-height: 1.45; +} + +.sgd-model-table tbody tr:last-child td { + border-bottom: none; +} + +.sgd-model-table tbody td:first-child { + width: 18%; + font-weight: 650; + color: rgb(31, 41, 55); +} + +.sgd-id-list { + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.sgd-id-list code { + display: inline-block; + max-width: 100%; + white-space: normal; + overflow-wrap: anywhere; + font-size: 0.78rem; + line-height: 1.4; +} + +.sgd-chip { + display: inline-flex; + align-items: center; + margin: 2px 4px 2px 0; + padding: 2px 8px; + border: 1px solid rgba(213, 88, 22, 0.22); + border-radius: 999px; + background: rgba(213, 88, 22, 0.08); + color: rgb(154, 52, 18); + font-size: 0.75rem; + font-weight: 650; + line-height: 1.45; +} + +.sgd-muted { + color: rgb(107, 114, 128); + font-size: 0.85rem; +} + +html.dark .sgd-model-table-wrap, +[data-theme="dark"] .sgd-model-table-wrap { + border-color: rgba(255, 255, 255, 0.12); + background: rgba(255, 255, 255, 0.025); +} + +html.dark .sgd-model-table thead th, +[data-theme="dark"] .sgd-model-table thead th { + border-bottom-color: rgba(213, 88, 22, 0.32); + background: rgba(213, 88, 22, 0.16) !important; + color: rgb(254, 215, 170); +} + +html.dark .sgd-model-table tbody td, +[data-theme="dark"] .sgd-model-table tbody td { + border-bottom-color: rgba(255, 255, 255, 0.08); +} + +html.dark .sgd-model-table tbody td:first-child, +[data-theme="dark"] .sgd-model-table tbody td:first-child { + color: rgb(243, 244, 246); +} + +html.dark .sgd-chip, +[data-theme="dark"] .sgd-chip { + border-color: rgba(251, 146, 60, 0.3); + background: rgba(251, 146, 60, 0.12); + color: rgb(253, 186, 116); +} + +html.dark .sgd-muted, +[data-theme="dark"] .sgd-muted { + color: rgb(156, 163, 175); +} + /* Bold text (**text**) */ .prose strong, .prose b { font-weight: 600; diff --git a/docs_new/docs.json b/docs_new/docs.json index f7391e5a6..93a039688 100644 --- a/docs_new/docs.json +++ b/docs_new/docs.json @@ -1204,7 +1204,6 @@ "docs/sglang-diffusion/installation", "docs/sglang-diffusion/compatibility_matrix", "docs/sglang-diffusion/disaggregation", - "docs/sglang-diffusion/quantization", { "group": "Usage", "pages": [ @@ -1218,31 +1217,34 @@ "pages": [ "docs/sglang-diffusion/performance-optimization", "docs/sglang-diffusion/deployment_cookbook", - "docs/sglang-diffusion/ring_sp_performance", "docs/sglang-diffusion/attention_backends", + "docs/sglang-diffusion/ring_sp_performance", + "docs/sglang-diffusion/dynamic_batching", { - "group": "Inference Batching", + "group": "Caching Acceleration", + "root": "docs/sglang-diffusion/caching-acceleration", + "tag": "approx", "pages": [ - "docs/sglang-diffusion/dynamic_batching" + "docs/sglang-diffusion/cache_dit", + "docs/sglang-diffusion/teacache" ] }, - "docs/sglang-diffusion/profiling", - "docs/sglang-diffusion/ci_perf" - ] - }, - { - "group": "Caching Strategies", - "pages": [ - "docs/sglang-diffusion/caching-acceleration", - "docs/sglang-diffusion/cache_dit", - "docs/sglang-diffusion/teacache" + "docs/sglang-diffusion/progressive_resolution", + "docs/sglang-diffusion/quantization", + "docs/sglang-diffusion/profiling" ] }, { "group": "References", "pages": [ - "docs/sglang-diffusion/environment_variables", + "docs/sglang-diffusion/environment_variables" + ] + }, + { + "group": "Development", + "pages": [ "docs/sglang-diffusion/support_new_models", + "docs/sglang-diffusion/ci_perf", "docs/sglang-diffusion/contributing" ] } diff --git a/docs_new/docs/sglang-diffusion/api/cli.mdx b/docs_new/docs/sglang-diffusion/api/cli.mdx index a8dd9e642..87d8fe899 100644 --- a/docs_new/docs/sglang-diffusion/api/cli.mdx +++ b/docs_new/docs/sglang-diffusion/api/cli.mdx @@ -83,7 +83,7 @@ Use `sglang generate --help` and `sglang serve --help` for the full argument lis - `--sp-degree {N}`: sequence parallelism size - `--ulysses-degree {N}` and `--ring-degree {N}`: USP parallelism controls - `--enable-cfg-parallel {true|false}`: enable or explicitly disable CFG parallelism -- `--attention-backend {BACKEND}`: attention backend for native SGLang pipelines +- `--attention-backend {BACKEND}`: attention backend for native SGLang and diffusers pipelines - `--component-attention-backends {MAP}`: per-component attention backend overrides, for example `text_encoder=torch_sdpa,transformer=fa` - `--attention-backend-config {CONFIG}`: attention backend configuration @@ -124,7 +124,7 @@ model_path: FastVideo/FastHunyuan-diffusers prompt: A beautiful woman in a red dress walking down a street output_path: outputs/ num_gpus: 2 -sp_size: 2 +sp_degree: 2 tp_size: 1 num_frames: 45 height: 720 @@ -262,7 +262,7 @@ Use `--backend diffusers` to force vanilla diffusers pipelines when no native SG Choose native SGLang, force native, or force diffusers - --diffusers-attention-backend + --attention-backend flash, _flash_3_hub, sage, xformers, native Attention backend for diffusers pipelines @@ -301,7 +301,7 @@ sglang generate \ --model-path AIDC-AI/Ovis-Image-7B \ --backend diffusers \ --trust-remote-code \ - --diffusers-attention-backend flash \ + --attention-backend flash \ --prompt "A serene Japanese garden with cherry blossoms" \ --height 1024 \ --width 1024 \ diff --git a/docs_new/docs/sglang-diffusion/attention_backends.mdx b/docs_new/docs/sglang-diffusion/attention_backends.mdx index 116f85abd..87117753c 100644 --- a/docs_new/docs/sglang-diffusion/attention_backends.mdx +++ b/docs_new/docs/sglang-diffusion/attention_backends.mdx @@ -1,6 +1,7 @@ --- title: "Attention Backends" description: "Select and configure attention backends for SGLang diffusion pipelines." +tag: "preserve" --- This document describes the attention backends available in sglang diffusion (`sglang.multimodal_gen`) and how to select them. diff --git a/docs_new/docs/sglang-diffusion/cache_dit.mdx b/docs_new/docs/sglang-diffusion/cache_dit.mdx index 2c6c88bc0..d568f1592 100644 --- a/docs_new/docs/sglang-diffusion/cache_dit.mdx +++ b/docs_new/docs/sglang-diffusion/cache_dit.mdx @@ -1,6 +1,7 @@ --- title: "Cache-DiT Acceleration" description: "Configure Cache-DiT acceleration for diffusion inference." +tag: "approx" --- SGLang integrates [Cache-DiT](https://github.com/vipshop/cache-dit), a caching acceleration engine for Diffusion Transformers (DiT), to achieve up to **1.69x inference speedup** with minimal quality loss. @@ -559,8 +560,7 @@ SGLang Diffusion x Cache-DiT supports almost all models originally supported in ## Limitations -- **SGLang-native pipelines**: Distributed support (TP/SP) is not yet validated; Cache-DiT will be automatically - disabled when `world_size > 1`. +- **SGLang-native pipelines**: Distributed Cache-DiT paths exist for supported pipelines. Hybrid SP+TP configurations add communication and cache coordination overhead, so validate them on the target model and hardware before using them as production defaults. - **SCM minimum steps**: SCM requires >= 8 inference steps to be effective - **Model support**: Only models registered in Cache-DiT's BlockAdapterRegister are supported diff --git a/docs_new/docs/sglang-diffusion/caching-acceleration.mdx b/docs_new/docs/sglang-diffusion/caching-acceleration.mdx index d0f3e73df..69a12f2c2 100644 --- a/docs_new/docs/sglang-diffusion/caching-acceleration.mdx +++ b/docs_new/docs/sglang-diffusion/caching-acceleration.mdx @@ -1,6 +1,7 @@ --- title: "Caching Acceleration" description: "Compare caching acceleration strategies for diffusion models." +tag: "approx" --- SGLang provides two complementary caching strategies for Diffusion Transformer (DiT) models. Both reduce denoising cost by skipping redundant computation, but they operate at different levels. @@ -70,13 +71,14 @@ See [teacache.md](./teacache) for detailed documentation. - Tracks L1 distance between modulated inputs across timesteps - When accumulated distance is below threshold, reuses cached residual -- Supports CFG with separate positive/negative caches +- Uses separate positive/negative caches for supported CFG model families ### Supported Models -- Wan (wan2.1, wan2.2) -- Hunyuan (HunyuanVideo) +- Wan2.1 - Z-Image +- Wan2.2: coefficients are not calibrated yet; enabling TeaCache is accepted but currently no-ops +- HunyuanVideo: not supported yet For Flux and Qwen models, TeaCache is automatically disabled when CFG is enabled. diff --git a/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx b/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx index e70e31ca1..1eeb840d1 100644 --- a/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx +++ b/docs_new/docs/sglang-diffusion/compatibility_matrix.mdx @@ -1,20 +1,182 @@ --- -title: "Supported Models" -description: "Check model compatibility across diffusion optimizations and backends." +title: "Supported Models and Optimization Compatibility" +description: "Check supported SGLang Diffusion models and their optimization compatibility." --- -The table below shows every supported model and the optimizations supported for them. +This page tracks supported SGLang Diffusion model families and their optimization compatibility. It also covers long-tail models that do not yet have dedicated cookbook recipes. -The symbols used have the following meanings: +For model-specific usage recipes, start from the [Diffusion Cookbook](/cookbook/diffusion/intro). Cookbook pages cover the primary models with examples; this page keeps the compact support and compatibility inventory. + +## Supported model inventory + +Pass the `Hugging Face Model ID` to `--model-path` for `sglang generate` or `sglang serve`. Python API users can pass the same ID to SGLang Diffusion model-loading helpers. + +Missing checkpoint aliases do not imply that a model family is unsupported. The runtime registry may also accept detector-based aliases or local model directories that match the same family. + +Rows are grouped when a family shares the same runtime path or optimization support. Use the detailed matrix below when you need per-optimization compatibility. + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model familyModel IDs
FLUX
black-forest-labs/FLUX.1-devblack-forest-labs/FLUX.2-devblack-forest-labs/FLUX.2-dev-NVFP4black-forest-labs/FLUX.2-klein-4Bblack-forest-labs/FLUX.2-klein-9Bblack-forest-labs/FLUX.2-klein-base-4Bblack-forest-labs/FLUX.2-klein-base-9B
Z-Image
Tongyi-MAI/Z-ImageTongyi-MAI/Z-Image-Turbo
Qwen-Image
Qwen/Qwen-ImageQwen/Qwen-Image-2512Qwen/Qwen-Image-EditQwen/Qwen-Image-Edit-2509Qwen/Qwen-Image-Edit-2511Qwen/Qwen-Image-Layered
SD3 / SD3.5
stabilityai/stable-diffusion-3-mediumstabilityai/stable-diffusion-3-medium-diffusersstabilityai/stable-diffusion-3.5-mediumstabilityai/stable-diffusion-3.5-medium-diffusersstabilityai/stable-diffusion-3.5-largestabilityai/stable-diffusion-3.5-large-diffusers
SANA
Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusersEfficient-Large-Model/SANA1.5_4.8B_1024px_diffusersEfficient-Large-Model/Sana_1600M_1024px_diffusersEfficient-Large-Model/Sana_600M_1024px_diffusersEfficient-Large-Model/Sana_1600M_512px_diffusersEfficient-Large-Model/Sana_600M_512px_diffusers
FireRed-Image
FireRedTeam/FireRed-Image-Edit-1.0FireRedTeam/FireRed-Image-Edit-1.1
JoyAI-Image
jdopensource/JoyAI-Image-Edit-Diffusers
Other image pipelines
zai-org/GLM-Imagetencent/Hunyuan3D-2baidu/ERNIE-Imagebaidu/ERNIE-Image-Turboideogram-ai/ideogram-4-fp8ideogram-ai/ideogram-4-nf4Comfy-Org/Ideogram-4
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model familyModel IDsResolution / modeOptimization support
FastWan
FastVideo/FastWan2.1-T2V-1.3B-DiffusersFastVideo/FastWan2.2-TI2V-5B-FullAttn-DiffusersFastVideo/FastWan2.2-TI2V-5B-Diffusers
480p / 720pVSA
Wan2.2
Wan-AI/Wan2.2-TI2V-5B-DiffusersWan-AI/Wan2.2-T2V-A14B-Diffusersnvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4Wan-AI/Wan2.2-I2V-A14B-Diffusers
TI2V / T2V / I2V, 480p / 720pSageLaserBSARain Fusion
HunyuanVideo
hunyuanvideo-community/HunyuanVideoFastVideo/FastHunyuan-diffusers
720×1280 / 544×960TileSageSVG2
Wan2.1
Wan-AI/Wan2.1-T2V-1.3B-DiffusersWan-AI/Wan2.1-T2V-14B-DiffusersWan-AI/Wan2.1-I2V-14B-480P-DiffusersWan-AI/Wan2.1-I2V-14B-720P-Diffusers
T2V / I2V, 480p / 720pTeaCacheTileSageSVG2LaserBSARain Fusion
TurboWan
IPostYellow/TurboWan2.1-T2V-1.3B-DiffusersIPostYellow/TurboWan2.1-T2V-14B-DiffusersIPostYellow/TurboWan2.1-T2V-14B-720P-DiffusersIPostYellow/TurboWan2.2-I2V-A14B-Diffusers
480p / 720pTeaCacheSLASageSLA
MOVA
OpenMOSS-Team/MOVA-360pOpenMOSS-Team/MOVA-720p
Video-audio, 360p / 720p; local MOVA detector aliases are also supported.No dedicated optimization listed
Wan2.1 Fun
weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers
480p inpaintingTeaCacheTileSageSVG2
Helios
BestWishYsh/Helios-BaseBestWishYsh/Helios-MidBestWishYsh/Helios-Distilled
720pNo dedicated optimization listed
LTX-2
Lightricks/LTX-2Lightricks/LTX-2.3
One-stage, two-stage, TI2V, HQNo dedicated optimization listed
Cosmos3
nvidia/Cosmos3-Nanonvidia/Cosmos3-Supernvidia/Cosmos3-Super-Text2Imagenvidia/Cosmos3-Super-Image2Video
T2V / I2V / T2INo dedicated optimization listed
+
+ + +
+ + + + + + + + + + + + + + + + + + + + +
Model familyModel IDs / detectorNotes
LingBotWorld
robbyant/lingbot-world-fast-diffusers
Realtime world model with causal state and control tokens.
SANA-WM
Efficient-Large-Model/SANA-WM_bidirectionalEfficient-Large-Model/SANA-WM_streaming
World-model pipeline with bidirectional and streaming checkpoints.
+
+ + + + +Wan2.2 TI2V 5B currently has known quality issues when used for I2V generation. + + +## Optimization compatibility + +The detailed video matrix uses these symbols: - ✅ = Full compatibility - ❌ = No compatibility - ⭕ = Does not apply to this model -## Models x Optimization - -The `HuggingFace Model ID` can be passed directly to `from_pretrained()` methods, and sglang-diffusion will use the -optimal -default parameters when initializing and generating videos. + ### Video Generation Models @@ -81,8 +243,8 @@ Optimization columns are abbreviated to keep the matrix readable: ❌ - FastWan2.2 TI2V 5B Full Attn - `FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers` + FastWan2.2 TI2V 5B + FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers
FastVideo/FastWan2.2-TI2V-5B-Diffusers 720p ⭕ ⭕ @@ -112,7 +274,7 @@ Optimization columns are abbreviated to keep the matrix readable: Wan2.2 T2V A14B - `Wan-AI/Wan2.2-T2V-A14B-Diffusers` + Wan-AI/Wan2.2-T2V-A14B-Diffusers
nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4 480p
720p ❌ ❌ @@ -397,7 +559,7 @@ Optimization columns are abbreviated to keep the matrix readable: Cosmos3-Super (T2V / I2V / T2I) - nvidia/Cosmos3-Super + nvidia/Cosmos3-Super
nvidia/Cosmos3-Super-Text2Image
nvidia/Cosmos3-Super-Image2Video 720p · 480p
1024×1024 (T2I) ❌ ❌ @@ -435,135 +597,7 @@ Optimization columns are abbreviated to keep the matrix readable: `transformer/config.json` at load time. A single checkpoint serves T2V, I2V (`--image-path`), and T2I (`--num-frames 1`). -### Image Generation Models - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Model NameHuggingFace Model ID
FLUX.1-dev`black-forest-labs/FLUX.1-dev`
FLUX.2-dev`black-forest-labs/FLUX.2-dev`
FLUX.2-dev-NVFP4black-forest-labs/FLUX.2-dev-NVFP4
FLUX.2-Klein-4Bblack-forest-labs/FLUX.2-klein-4B
FLUX.2-Klein-9Bblack-forest-labs/FLUX.2-klein-9B
Z-ImageTongyi-MAI/Z-Image
Z-Image-TurboTongyi-MAI/Z-Image-Turbo
GLM-Imagezai-org/GLM-Image
Qwen ImageQwen/Qwen-Image
Qwen Image 2512Qwen/Qwen-Image-2512
Qwen Image Edit`Qwen/Qwen-Image-Edit`
Qwen Image Edit 2509Qwen/Qwen-Image-Edit-2509
Qwen Image Edit 2511Qwen/Qwen-Image-Edit-2511
Qwen Image LayeredQwen/Qwen-Image-Layered
SD3 Mediumstabilityai/stable-diffusion-3-medium-diffusers
SD3.5 Mediumstabilityai/stable-diffusion-3.5-medium-diffusers
SD3.5 Largestabilityai/stable-diffusion-3.5-large-diffusers
Hunyuan3D-2tencent/Hunyuan3D-2
SANA 1.5 1.6BEfficient-Large-Model/SANA1.5_1.6B_1024px_diffusers
SANA 1.5 4.8BEfficient-Large-Model/SANA1.5_4.8B_1024px_diffusers
SANA 1600M 1024pxEfficient-Large-Model/Sana_1600M_1024px_diffusers
SANA 600M 1024pxEfficient-Large-Model/Sana_600M_1024px_diffusers
SANA 1600M 512pxEfficient-Large-Model/Sana_1600M_512px_diffusers
SANA 600M 512pxEfficient-Large-Model/Sana_600M_512px_diffusers
FireRed-Image-Edit 1.0FireRedTeam/FireRed-Image-Edit-1.0
FireRed-Image-Edit 1.1FireRedTeam/FireRed-Image-Edit-1.1
ERNIE-Imagebaidu/ERNIE-Image
ERNIE-Image-Turbobaidu/ERNIE-Image-Turbo
+ ## Supported Components @@ -599,7 +633,8 @@ Use the component name from the pipeline's `model_index.json` or the native pipe - + + diff --git a/docs_new/docs/sglang-diffusion/deployment_cookbook.mdx b/docs_new/docs/sglang-diffusion/deployment_cookbook.mdx index 91a2ddbe2..454a22bc1 100644 --- a/docs_new/docs/sglang-diffusion/deployment_cookbook.mdx +++ b/docs_new/docs/sglang-diffusion/deployment_cookbook.mdx @@ -1,9 +1,10 @@ --- -title: "Deployment Cookbook" -description: "Practical defaults for choosing CPU offload, FSDP, CFG parallelism, SP, and TP in SGLang Diffusion." +title: "Deployment and Performance Modes" +description: "Choose CPU offload, FSDP, CFG parallelism, SP, TP, and performance-mode presets in SGLang Diffusion." +tag: "preserve" --- -This page gives practical defaults for choosing CPU offload, FSDP, CFG parallelism, SP, and TP. +This page gives practical defaults for choosing `--performance-mode`, CPU offload, FSDP, CFG parallelism, SP, and TP. ## Quick Rule diff --git a/docs_new/docs/sglang-diffusion/disaggregation.mdx b/docs_new/docs/sglang-diffusion/disaggregation.mdx index fcdc4e679..e55806a20 100644 --- a/docs_new/docs/sglang-diffusion/disaggregation.mdx +++ b/docs_new/docs/sglang-diffusion/disaggregation.mdx @@ -269,9 +269,13 @@ Set `--disagg-p2p-hostname` to the actual IP on each machine. For multi-machine, + + + + - +
--denoiser-tp / --denoiser-sp / --denoiser-ulysses / --denoiser-ring Denoiser parallelism
--decoder-spDecoder sequence parallelism
--decoder-tpDecoder tensor parallelismDeprecated alias for --decoder-sp
diff --git a/docs_new/docs/sglang-diffusion/dynamic_batching.mdx b/docs_new/docs/sglang-diffusion/dynamic_batching.mdx index 795536a49..a4d5d2640 100644 --- a/docs_new/docs/sglang-diffusion/dynamic_batching.mdx +++ b/docs_new/docs/sglang-diffusion/dynamic_batching.mdx @@ -1,6 +1,7 @@ --- title: "Inference Batching" description: "Batch compatible native SGLang-Diffusion requests during serving." +tag: "preserve" mode: wide --- Dynamic batching is an opt-in SGLang-Diffusion serving mode that merges compatible queued requests into one native pipeline batch. It is separate from LLM continuous batching and tokenizer batching. @@ -40,7 +41,7 @@ Use `--batching-config /path/to/batching_config.json` to load JSON rules when a ## Compatibility -An initial implementation of dynamic batching for T2I and T2V models can be found in [#18764](https://github.com/sgl-project/sglang/pull/18764). The current compatibility grid is below and will be updated as more coverage is added. See [Supported Models](./compatibility_matrix) for full model IDs. +An initial implementation of dynamic batching for T2I and T2V models can be found in [#18764](https://github.com/sgl-project/sglang/pull/18764). The current compatibility grid is below and will be updated as more coverage is added. See [Supported Models and Optimization Compatibility](./compatibility_matrix) for common model IDs and optimization support. `✅` means supported, `❌` means not currently supported, `?` means untested, and `-` means not applicable. diff --git a/docs_new/docs/sglang-diffusion/index.mdx b/docs_new/docs/sglang-diffusion/index.mdx index 42a9ffbe1..3fdea40a3 100644 --- a/docs_new/docs/sglang-diffusion/index.mdx +++ b/docs_new/docs/sglang-diffusion/index.mdx @@ -30,11 +30,10 @@ sglang serve --model-path Qwen/Qwen-Image --port 30010 ## Start Here - [Installation](/docs/sglang-diffusion/installation): install SGLang Diffusion and platform dependencies -- [Compatibility Matrix](/docs/sglang-diffusion/compatibility_matrix): check model, optimization, and component override support +- [Supported Models and Optimization Compatibility](/docs/sglang-diffusion/compatibility_matrix): check supported model families, long-tail coverage, and optimization support - [CLI](/docs/sglang-diffusion/api/cli): run one-off generation jobs or launch a persistent server - [OpenAI-Compatible API](/docs/sglang-diffusion/api/openai_api): send image and video requests to the HTTP server -- [Attention Backends](/docs/sglang-diffusion/attention_backends): choose the best backend for your model and hardware -- [Inference Batching](/docs/sglang-diffusion/dynamic_batching): batch compatible native diffusion requests during serving +- [Performance Overview](/docs/sglang-diffusion/performance-optimization): choose speed, memory, parallelism, caching, and quality-tradeoff levers - [Caching Acceleration](/docs/sglang-diffusion/caching-acceleration): use Cache-DiT or TeaCache to reduce denoising cost - [Quantization](/docs/sglang-diffusion/quantization): load quantized transformer checkpoints - [Contributing](/docs/sglang-diffusion/contributing): contribution workflow, adding new models, and CI perf baselines @@ -42,10 +41,17 @@ sglang serve --model-path Qwen/Qwen-Image --port 30010 ## Additional Documentation - [Post-Processing](/docs/sglang-diffusion/api/post_processing): frame interpolation and upscaling -- [Performance Overview](/docs/sglang-diffusion/performance-optimization): overview of attention, caching, and profiling +- [Deployment and Performance Modes](/docs/sglang-diffusion/deployment_cookbook): choose `--performance-mode`, offload, FSDP, CFG parallelism, SP, and TP +- [Attention Backends](/docs/sglang-diffusion/attention_backends): choose the best backend for your model and hardware +- [Sequence Parallelism](/docs/sglang-diffusion/ring_sp_performance): configure SP, Ulysses, and ring-based splitting for long sequences +- [Inference Batching](/docs/sglang-diffusion/dynamic_batching): batch compatible native diffusion requests during serving +- [Progressive Resolution Generation](/docs/sglang-diffusion/progressive_resolution): run early denoising steps at lower latent resolution for selected pipelines - [Environment Variables](/docs/sglang-diffusion/environment_variables): platform, caching, storage, and debugging configuration + +## Developer Documentation + - [Support New Models](/docs/sglang-diffusion/support_new_models): implementation guide for new diffusion pipelines -- [CI Performance](/docs/sglang-diffusion/ci_perf): performance baseline generation +- [CI Performance Baselines](/docs/sglang-diffusion/ci_perf): generate and update performance baselines used in CI ## References diff --git a/docs_new/docs/sglang-diffusion/installation.mdx b/docs_new/docs/sglang-diffusion/installation.mdx index ec1b0049f..f54f754c7 100644 --- a/docs_new/docs/sglang-diffusion/installation.mdx +++ b/docs_new/docs/sglang-diffusion/installation.mdx @@ -57,7 +57,7 @@ docker run --gpus all \ ## Platform-Specific: ROCm (AMD GPUs) -For AMD Instinct GPUs (e.g., MI300X), you can use the ROCm-enabled Docker image: +For AMD Instinct GPUs (e.g., MI300X), use a ROCm-enabled Docker image from [lmsysorg/sglang](https://hub.docker.com/r/lmsysorg/sglang). The tag below is an example for ROCm 7.0 on MI300X and may lag the latest release tag: ```bash Command docker run --device=/dev/kfd --device=/dev/dri --ipc=host \ @@ -71,7 +71,7 @@ For detailed ROCm system configuration and installation from source, see [AMD GP ## Platform-Specific: MUSA (Moore Threads GPUs) -For Moore Threads GPUs (MTGPU) with the MUSA software stack, please follow the instructions below to install from source: +For Moore Threads GPUs (MTGPU) with the MUSA software stack, follow the platform guide first. If the source tree still requires the alternate platform `pyproject` fallback, keep a backup of the default file before switching: ```bash Command # Clone the repository @@ -80,7 +80,8 @@ cd sglang # Install the Python packages pip install --upgrade pip -rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +mv python/pyproject.toml python/pyproject.toml.bak +cp python/pyproject_other.toml python/pyproject.toml pip install -e "python[all_musa]" ``` @@ -106,7 +107,7 @@ sglang generate --model-path black-forest-labs/FLUX.1-dev \ ## Platform-Specific: Apple MPS -For Apple MPS, please follow the instructions below to install from source: +For Apple MPS, follow the instructions below to install from source. If the source tree still requires the alternate platform `pyproject` fallback, keep a backup of the default file before switching: ```bash Command # Install ffmpeg @@ -125,6 +126,7 @@ source sglang-diffusion/bin/activate # Install the Python packages uv pip install --upgrade pip -rm -f python/pyproject.toml && mv python/pyproject_other.toml python/pyproject.toml +mv python/pyproject.toml python/pyproject.toml.bak +cp python/pyproject_other.toml python/pyproject.toml uv pip install -e "python[all_mps]" ``` diff --git a/docs_new/docs/sglang-diffusion/performance-optimization.mdx b/docs_new/docs/sglang-diffusion/performance-optimization.mdx index ef5a8950f..0b604bf3c 100644 --- a/docs_new/docs/sglang-diffusion/performance-optimization.mdx +++ b/docs_new/docs/sglang-diffusion/performance-optimization.mdx @@ -1,73 +1,126 @@ --- title: "Performance Optimization" -description: "Optimize SGLang diffusion performance with caching, kernels, and profiling." +description: "Choose performance levers for SGLang Diffusion by latency, throughput, memory, and quality tradeoffs." --- -This section covers the main performance levers for SGLang Diffusion: attention backends, caching acceleration, and profiling. -## Overview +Use this page as the starting point for SGLang Diffusion performance work. It separates performance levers into two decision classes: + +- **Output-preserving / lossless-style:** system settings that should preserve model behavior while changing residency, parallelism, kernels, or scheduling. +- **Quality-tradeoff / lossy or approximate:** techniques that can change the denoising path, numerical representation, or generated output. + +The docs use "output-preserving" instead of promising bit-exact "lossless" because different kernels, GPU types, or precision paths can still introduce small numerical differences. The decision boundary is whether the optimization intentionally trades quality or output equivalence for speed. + +## Start Here + +1. Pick a serving or generation mode from [Deployment and Performance Modes](./deployment_cookbook). `--performance-mode auto` is the default; use `speed` when the model fits in GPU memory and latency matters most, `memory` when GPU memory is the bottleneck, and `manual` when every performance flag should be explicit. +2. Choose the right attention backend from [Attention Backends](./attention_backends). +3. Use [Sequence Parallelism](./ring_sp_performance) only when the model and video shape benefit from sequence splitting. +4. Use [Inference Batching](./dynamic_batching) for concurrent compatible requests during serving. +5. Use [Profiling](./profiling) before changing several levers at once. + +## Output-Preserving / Lossless-Style Levers + +These settings should preserve model behavior while changing residency, parallelism, kernels, or scheduling. They are the first choices for production tuning. - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + +
OptimizationTypeDescriptionLeverUse whenDocs
Cache-DiTCachingBlock-level caching with DBCache, TaylorSeer, and SCM--performance-modeYou want a safe preset for speed or memory without overriding explicit flags.Deployment and Performance Modes
TeaCacheCachingTimestep-level caching based on temporal similarityOffload, FSDP, CFG parallelismGPU memory, multi-GPU residency, or CFG branch splitting is the main bottleneck.Deployment and Performance Modes
Attention BackendsKernelOptimized attention implementations (FlashAttention, SageAttention, etc.)Sequence parallelismLong image/video sequences need sequence-level parallelism.Sequence Parallelism
Inference BatchingSchedulerRequest batching for native diffusion servingAttention backendKernel choice dominates DiT latency or memory.Attention Backends
ProfilingDiagnosticsPyTorch Profiler and Nsight Systems guidanceDynamic batchingServing many compatible requests concurrently.Inference Batching
-## Start Here +## Quality-Tradeoff / Lossy Or Approximate Levers -- Use [Attention Backends](./attention_backends) to choose the best backend for your model and hardware. -- Use [Inference Batching](./dynamic_batching) to improve throughput for compatible concurrent requests. -- Use [Caching Acceleration](./caching-acceleration) to reduce denoising cost with Cache-DiT or TeaCache. -- Use [Profiling](./profiling) when you need to diagnose a bottleneck rather than guess. +These techniques can change the denoising path, numerical representation, or generated output. They are useful after you have a baseline and an acceptance criterion for quality. -## Caching at a Glance + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LeverTradeoffDocs
Cache-DiTSkips selected DiT block or step computation based on cache decisions.Cache-DiT
TeaCacheReuses residuals when consecutive denoising steps are similar enough.TeaCache
Progressive resolutionRuns early denoising at lower latent resolution for supported pipelines.Progressive Resolution Generation
QuantizationUses lower-precision transformer weights or activations.Quantization
-- [Cache-DiT](./cache_dit) is block-level caching for diffusers pipelines and higher speedup-oriented tuning. -- [TeaCache](./teacache) is timestep-level caching built into SGLang model families. +## Practical Order +1. Establish a baseline with the target model, resolution, frame count, step count, and GPU type. +2. Select `--performance-mode` and explicit residency or parallelism flags. +3. Tune attention backend and batching for the deployment pattern. +4. Profile if the bottleneck is unclear. +5. Add caching, progressive resolution, or quantization only after comparing output quality against your acceptance target. -## Current Baseline Snapshot +## Diagnostics -For Ring SP benchmark details, see: - -- [Ring SP Performance](./ring_sp_performance) +[Profiling](./profiling) is not an optimization technique by itself. It belongs in the performance workflow because it tells you which stage, kernel, or denoising step is worth optimizing before you change multiple levers. ## References -- [Cache-DiT Repository](https://github.com/vipshop/cache-dit) -- [TeaCache Paper](https://arxiv.org/abs/2411.14324) +- [Deployment and Performance Modes](./deployment_cookbook) +- [Attention Backends](./attention_backends) +- [Sequence Parallelism](./ring_sp_performance) +- [Caching Strategies](./caching-acceleration) +- [Profiling](./profiling) diff --git a/docs_new/docs/sglang-diffusion/profiling.mdx b/docs_new/docs/sglang-diffusion/profiling.mdx index 2fb327a2a..71f34633a 100644 --- a/docs_new/docs/sglang-diffusion/profiling.mdx +++ b/docs_new/docs/sglang-diffusion/profiling.mdx @@ -1,6 +1,7 @@ --- title: "Profiling" description: "Profile SGLang diffusion workloads with PyTorch Profiler and Nsight Systems." +tag: "diag" --- This guide covers profiling techniques for multimodal generation pipelines in SGLang. diff --git a/docs_new/docs/sglang-diffusion/progressive_resolution.mdx b/docs_new/docs/sglang-diffusion/progressive_resolution.mdx index a7b71992e..f38fc8d24 100644 --- a/docs_new/docs/sglang-diffusion/progressive_resolution.mdx +++ b/docs_new/docs/sglang-diffusion/progressive_resolution.mdx @@ -1,19 +1,18 @@ --- title: "Progressive Resolution Generation" description: "Experimental spectral progressive resolution growing for selected SGLang Diffusion pipelines." +tag: "approx" --- Progressive resolution growing is an experimental feature for selected SGLang Diffusion pipelines. It runs early denoising steps at a coarser latent resolution and spectrally upsamples the latent before the full-resolution steps. On the benchmark setup below, this reduces the quadratic attention cost of the DiT transformer and yields up to **1.63× speedup on FLUX.1**, **1.93× speedup on FLUX.2**, **2.33× speedup on Z-Image**, **2.78× speedup on Wan 2.1 T2V**, and **1.69× speedup on Qwen-Image**. -This page is intentionally not linked from the main documentation navigation while the feature is still experimental. - Based on [Spectral Progressive Diffusion (arXiv 2605.18736)](https://arxiv.org/abs/2605.18736). ## Overview DiT attention is O(n²) in sequence length. Running the first N denoising steps at half the spatial resolution cuts the attention cost to ~6% for those steps. -The transition point — how many steps to run at each resolution — is computed from the **Bayes-optimal frequency-activation criterion**: frequencies that cannot be resolved at the coarse scale are not denoised there. This makes the speedup lossless by construction. +The transition point — how many steps to run at each resolution — is computed from the **Bayes-optimal frequency-activation criterion**: frequencies that cannot be resolved at the coarse scale are not denoised there. The method is designed to preserve quality under this criterion, but generated outputs can still differ from the full-resolution baseline. | Model | Full-res tokens | Half-res tokens | Token-step ratio | |-------|----------------|----------------|-----------------| diff --git a/docs_new/docs/sglang-diffusion/quantization.mdx b/docs_new/docs/sglang-diffusion/quantization.mdx index 4c75a61c5..9ec8a70e0 100644 --- a/docs_new/docs/sglang-diffusion/quantization.mdx +++ b/docs_new/docs/sglang-diffusion/quantization.mdx @@ -1,5 +1,6 @@ --- title: "Quantization" +tag: "approx" metatags: description: "SGLang-Diffusion supports quantized transformer checkpoints. In most cases, keep the base model and the quantized transformer override separate." --- diff --git a/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx b/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx index eca61967a..7844b86ec 100644 --- a/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx +++ b/docs_new/docs/sglang-diffusion/ring_sp_performance.mdx @@ -1,46 +1,108 @@ --- -title: "Ring SP Benchmark: Wan2.2-TI2V-5B (u1r2 vs Baseline)" +title: "Sequence Parallelism" +tag: "preserve" metatags: - description: "Review Ring-SP benchmark results for Wan2.2-TI2V-5B-Diffusers in SGLang Diffusion." + description: "Configure sequence parallelism, Ulysses, and ring-based sequence splitting for SGLang Diffusion workloads." --- -This page reports Ring-SP performance for `Wan2.2-TI2V-5B-Diffusers` using: +Sequence parallelism splits long image or video latent sequences across GPUs. In SGLang Diffusion, the public controls are: -- Parallel config: `sp=2, ulysses=1, ring=2` (short: `u1r2`) -- Baseline config: `sp=1, ulysses=1, ring=1` (short: `u1r1`) +- `--sp-degree`: total sequence parallel degree +- `--ulysses-degree`: Ulysses parallel degree +- `--ring-degree`: ring parallel degree -## Benchmark Setup +The degrees must satisfy: -- Model: `Wan2.2-TI2V-5B-Diffusers` -- GPU: `48G RTX40 series * 2` +```text +sp_degree = ulysses_degree * ring_degree +``` -## Online Serving +Use SP when sequence length or video shape makes the DiT forward pass the bottleneck and the model supports sequence sharding. For latency-oriented multi-GPU Qwen/Wan deployments, also compare against CFG parallelism and FSDP; SP is not automatically the best multi-GPU setting for every model. -### Ring SP (`u1r2`) +## Recommended Commands + +### Two-GPU Sequence Parallelism + +This example uses two GPUs with `sp=2`, `ulysses=1`, and `ring=2`. ```bash sglang serve \ - --model-type diffusion \ - --model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \ - --num-gpus 2 --sp-degree 2 --ulysses-degree 1 --ring-degree 2 \ + --model-path Wan-AI/Wan2.2-TI2V-5B-Diffusers \ + --num-gpus 2 \ + --sp-degree 2 \ + --ulysses-degree 1 \ + --ring-degree 2 \ --port 8898 ``` -### Baseline (`u1r1`) +### Single-GPU Baseline + +Use an explicit single-GPU baseline before attributing a gain to sequence parallelism. ```bash sglang serve \ - --model-type diffusion \ - --model-path /model/HuggingFace/Wan-AI/Wan2.2-TI2V-5B-Diffusers \ - --num-gpus 1 --sp-degree 1 --ulysses-degree 1 --ring-degree 1 \ + --model-path Wan-AI/Wan2.2-TI2V-5B-Diffusers \ + --num-gpus 1 \ + --sp-degree 1 \ + --ulysses-degree 1 \ + --ring-degree 1 \ --port 8898 ``` -## Benchmarks +## Choosing The Degrees -### Benchmark Disclaimer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SettingTypical useNotes
--sp-degree 1Single-GPU or no sequence splittingUse this as the baseline.
--ulysses-degree NUlysses-only sequence parallelismWhen ring parallelism is not needed, keep --ring-degree 1.
--ring-degree NRing-based sequence splitting over long sequencesKeep --sp-degree equal to ulysses_degree * ring_degree.
-These benchmarks are provided for reference under one specific setup and command configuration. Actual performance may vary with model settings, runtime environment, and request patterns. +## Benchmarking Guidance + +When benchmarking SP, compare the same model, precision, resolution, frame count, step count, scheduler settings, prompt type, and output path. Report both stage latency and peak GPU memory; SP can reduce per-GPU memory while adding communication overhead. + +Useful metrics: + +- End-to-end latency +- Denoising stage latency +- Decoding stage latency +- Peak GPU memory and peak allocated memory +- Communication or runtime overhead when available + +## Reference Benchmark + +The following numbers are a reference measurement for one setup. They are not a general promise for all Wan2.2 deployments. + +- Model: `Wan-AI/Wan2.2-TI2V-5B-Diffusers` +- Hardware: two 48 GB RTX 40-series GPUs for sequence parallelism, one 48 GB RTX 40-series GPU for baseline +- Sequence parallel config: `sp=2, ulysses=1, ring=2` (`u1r2`) +- Baseline config: `sp=1, ulysses=1, ring=1` (`u1r1`) ### Stage Time Breakdown @@ -150,9 +212,4 @@ These benchmarks are provided for reference under one specific setup and command -## Summary - -- End-to-end latency improves from `90.63s` to `63.74s` (`1.42x`). -- Main gains come from `Denoising` (`1.36x`) and `Decoding` (`1.75x`). -- Absolute memory usage drops noticeably on Ring-SP (`Peak GPU Memory -7.33GB`, `Peak Allocated -7.05GB`). -- Overhead ratio rises (`+7.9pp`), so future tuning can focus on reducing communication/runtime overhead while preserving the latency gain. +In this setup, end-to-end latency improved from `90.63s` to `63.74s` (`1.42x`) and peak GPU memory dropped by `7.33GB`. The overhead ratio increased, so future tuning should still check communication and runtime overhead on the target hardware. diff --git a/docs_new/docs/sglang-diffusion/teacache.mdx b/docs_new/docs/sglang-diffusion/teacache.mdx index d7f86219f..e96efc1a9 100644 --- a/docs_new/docs/sglang-diffusion/teacache.mdx +++ b/docs_new/docs/sglang-diffusion/teacache.mdx @@ -1,6 +1,7 @@ --- title: "TeaCache Acceleration" description: "Configure TeaCache for temporal similarity-based diffusion acceleration." +tag: "approx" --- > **Note**: This is one of two caching strategies available in SGLang. @@ -14,7 +15,7 @@ TeaCache works by: 1. Tracking the L1 distance between modulated inputs across consecutive timesteps 2. Accumulating the rescaled L1 distance over steps 3. When accumulated distance is below a threshold, reusing the cached residual -4. Supporting CFG (Classifier-Free Guidance) with separate positive/negative caches +4. Using separate positive/negative caches for supported CFG model families ## How It Works @@ -39,11 +40,11 @@ accumulated += poly(coefficients)(rel_l1) ### CFG Support -For models that support CFG cache separation (Wan, Hunyuan, Z-Image), TeaCache maintains separate caches for positive and negative branches: +For models that support CFG cache separation, TeaCache maintains separate caches for positive and negative branches: - `previous_modulated_input` / `previous_residual` for positive branch - `previous_modulated_input_negative` / `previous_residual_negative` for negative branch -For models that don't support CFG separation (Flux, Qwen), TeaCache is automatically disabled when CFG is enabled. +For models that do not support CFG separation, TeaCache is automatically disabled when CFG is enabled. ## Configuration @@ -77,7 +78,7 @@ params = TeaCacheParams( `teacache_thresh` float - Threshold for accumulated L1 distance. Lower = more caching, faster but potentially lower quality + Threshold for accumulated L1 distance. Higher = more caching, faster but potentially lower quality `coefficients` @@ -93,7 +94,7 @@ Different models may have different optimal configurations. The coefficients are ## Supported Models -TeaCache is built into the following model families: +TeaCache support status by model family: @@ -110,19 +111,24 @@ TeaCache is built into the following model families: - + - + - + - + + + + + +
Wan (wan2.1, wan2.2)Wan2.1 Yes Full support
Hunyuan (HunyuanVideo)Wan2.2 YesTo be supportedCoefficients are not calibrated yet; enabling TeaCache is accepted but currently no-ops
Z-Image YesTo be supportedFull support
HunyuanVideoNoNot supported yet
Flux