From 0caf0fc01dc9fdc0e96732815f7cc644cbc1e2de Mon Sep 17 00:00:00 2001 From: Andrew Kuksa <149417153+andrew52522@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:50:15 +0300 Subject: [PATCH] [Diffusion][Docs] Ascend A2, A3 add basic usage and benchmark results in diffusion cookbook (#30614) Co-authored-by: ANDREW_K --- docs_new/cookbook/diffusion/FLUX/FLUX.mdx | 224 ++++++++++----- .../diffusion/Qwen-Image/Qwen-Image.mdx | 207 ++++++++++---- docs_new/cookbook/diffusion/Wan/Wan2.1.mdx | 263 ++++++++++++----- docs_new/cookbook/diffusion/Wan/Wan2.2.mdx | 265 +++++++++++++----- .../diffusion/Z-Image/Z-Image-Turbo.mdx | 205 ++++++++++---- .../snippets/diffusion/flux-deployment.jsx | 38 ++- .../diffusion/qwen-image-deployment.jsx | 35 ++- .../snippets/diffusion/wan21-deployment.jsx | 61 +++- .../snippets/diffusion/wan22-deployment.jsx | 72 ++++- .../diffusion/zimage-turbo-deployment.jsx | 34 ++- 10 files changed, 1066 insertions(+), 338 deletions(-) diff --git a/docs_new/cookbook/diffusion/FLUX/FLUX.mdx b/docs_new/cookbook/diffusion/FLUX/FLUX.mdx index 840905699..79aff623b 100644 --- a/docs_new/cookbook/diffusion/FLUX/FLUX.mdx +++ b/docs_new/cookbook/diffusion/FLUX/FLUX.mdx @@ -44,7 +44,7 @@ This section provides deployment configurations optimized for different hardware FLUX models are optimized for high-quality image generation. The recommended launch configurations vary by hardware and model version. -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving FLUX on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform and model version. SGLang supports serving FLUX on NVIDIA B200, H200, H100, and AMD MI355X, MI325X, MI300X GPUs and Ascend A2, A3 NPUs. @@ -215,80 +215,174 @@ Test Environment: - Model: black-forest-labs/FLUX.1-dev - sglang diffusion version: 0.5.6.post2 -**Server Command**: + + + **Server Command**: -```shell Command -sglang serve --model-path black-forest-labs/FLUX.1-dev --port 30000 -``` + ```shell Command + sglang serve --model-path black-forest-labs/FLUX.1-dev --port 30000 + ``` -**Benchmark Command**: + **Benchmark Command**: -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Backend: sglang-image -Model: black-forest-labs/FLUX.1-dev -Dataset: vbench -Task: t2v --------------------------------------------------- -Benchmark duration (s): 50.97 -Request rate: inf -Max request concurrency: 1 -Successful requests: 1/1 --------------------------------------------------- -Request throughput (req/s): 0.02 -Latency Mean (s): 50.9681 -Latency Median (s): 50.9681 -Latency P99 (s): 50.9681 --------------------------------------------------- -Peak Memory Max (MB): 27905.19 -Peak Memory Mean (MB): 27905.19 -Peak Memory Median (MB): 27905.19 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Backend: sglang-image + Model: black-forest-labs/FLUX.1-dev + Dataset: vbench + Task: t2v + -------------------------------------------------- + Benchmark duration (s): 50.97 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + -------------------------------------------------- + Request throughput (req/s): 0.02 + Latency Mean (s): 50.9681 + Latency Median (s): 50.9681 + Latency P99 (s): 50.9681 + -------------------------------------------------- + Peak Memory Max (MB): 27905.19 + Peak Memory Mean (MB): 27905.19 + Peak Memory Median (MB): 27905.19 + ============================================================ + ``` + + + **Server Command**: + + ```shell Command + #One A3 card has 2 npu chips + sglang serve --tp-size 2 --sp-degree 1 --model-path black-forest-labs/FLUX.1-dev --num-gpus 2 + ``` + + **Benchmark Command**: + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: black-forest-labs/FLUX.1-dev + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 16.30 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + Completed outputs: 1 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.06 + Output throughput (outputs/s): 0.06 + Latency Mean (s): 16.30 + Latency Median (s): 16.30 + Latency P90 (s): 16.30 + Latency P95 (s): 16.30 + Latency P99 (s): 16.30 + -------------------------------------------------- + Peak Memory Max (MB): 19972.00 + Peak Memory Mean (MB): 19972.00 + Peak Memory Median (MB): 19972.00 + ------------------------------------------------------------ + ``` + + #### 5.1.2 Generate images with high concurrency -**Server Command** : + + + **Server Command** : -```shell Command -sglang serve --model-path black-forest-labs/FLUX.1-dev --port 30000 -``` + ```shell Command + sglang serve --model-path black-forest-labs/FLUX.1-dev --port 30000 + ``` -**Benchmark Command** : + **Benchmark Command** : -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-image --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-image --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20 + ``` -**Result** : + **Result** : -```text Output -================= Serving Benchmark Result ================= -Backend: sglang-image -Model: black-forest-labs/FLUX.1-dev -Dataset: vbench -Task: t2v --------------------------------------------------- -Benchmark duration (s): 111.79 -Request rate: inf -Max request concurrency: 20 -Successful requests: 20/20 --------------------------------------------------- -Request throughput (req/s): 0.18 -Latency Mean (s): 67.0646 -Latency Median (s): 66.9691 -Latency P99 (s): 110.8949 --------------------------------------------------- -Peak Memory Max (MB): 27917.19 -Peak Memory Mean (MB): 27916.59 -Peak Memory Median (MB): 27917.19 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Backend: sglang-image + Model: black-forest-labs/FLUX.1-dev + Dataset: vbench + Task: t2v + -------------------------------------------------- + Benchmark duration (s): 111.79 + Request rate: inf + Max request concurrency: 20 + Successful requests: 20/20 + -------------------------------------------------- + Request throughput (req/s): 0.18 + Latency Mean (s): 67.0646 + Latency Median (s): 66.9691 + Latency P99 (s): 110.8949 + -------------------------------------------------- + Peak Memory Max (MB): 27917.19 + Peak Memory Mean (MB): 27916.59 + Peak Memory Median (MB): 27917.19 + ============================================================ + ``` + + + **Server Command** : + + ```shell Command + #One A3 card has 2 npu chips + sglang serve --tp-size 2 --sp-degree 1 --model-path black-forest-labs/FLUX.1-dev --num-gpus 2 + ``` + + **Benchmark Command** : + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 + ``` + + **Result** : + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: black-forest-labs/FLUX.1-dev + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 300.85 + Request rate: inf + Max request concurrency: 20 + Successful requests: 18/20 + Completed outputs: 18 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.06 + Output throughput (outputs/s): 0.06 + Latency Mean (s): 155.16 + Latency Median (s): 155.11 + Latency P90 (s): 266.30 + Latency P95 (s): 280.15 + Latency P99 (s): 291.23 + -------------------------------------------------- + Peak Memory Max (MB): 19972.00 + Peak Memory Mean (MB): 19972.00 + Peak Memory Median (MB): 19972.00 + ------------------------------------------------------------ + ``` + + diff --git a/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx b/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx index 630da7056..8f1367ee9 100644 --- a/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx +++ b/docs_new/cookbook/diffusion/Qwen-Image/Qwen-Image.mdx @@ -24,7 +24,7 @@ This section provides deployment configurations optimized for different hardware ### 3.1 Basic Configuration -Qwen-Image is a text-to-image model. The recommended launch configurations vary by hardware. +Qwen-Image is a text-to-image model. The recommended launch configurations vary by hardware. SGLang supports serving Qwen-Image on NVIDIA B200, B300, H200, H100, AMD MI300X, MI325X, MI355X GPUs and Ascend A2, A3 NPUs. **Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform. @@ -221,73 +221,160 @@ Test Environment: #### 5.1.1 Generate an image -**Server Command**: + + + **Server Command**: -```shell Command -sglang serve --model-path Qwen/Qwen-Image \ - --ulysses-degree=1 --ring-degree=1 --port 30000 -``` + ```shell Command + sglang serve --model-path Qwen/Qwen-Image \ + --ulysses-degree=1 --ring-degree=1 --port 30000 + ``` -**Benchmark Command**: + **Benchmark Command**: -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-image --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-image --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Task: text-to-image -Model: Qwen/Qwen-Image -Dataset: vbench --------------------------------------------------- -Benchmark duration (s): 29.04 -Request rate: inf -Max request concurrency: 1 -Successful requests: 1/1 --------------------------------------------------- -Request throughput (req/s): 0.03 -Latency Mean (s): 29.0378 -Latency Median (s): 29.0378 -Latency P99 (s): 29.0378 --------------------------------------------------- -Peak Memory Max (MB): 48018.83 -Peak Memory Mean (MB): 48018.83 -Peak Memory Median (MB): 48018.83 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Qwen/Qwen-Image + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 29.04 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + -------------------------------------------------- + Request throughput (req/s): 0.03 + Latency Mean (s): 29.0378 + Latency Median (s): 29.0378 + Latency P99 (s): 29.0378 + -------------------------------------------------- + Peak Memory Max (MB): 48018.83 + Peak Memory Mean (MB): 48018.83 + Peak Memory Median (MB): 48018.83 + ============================================================ + ``` + + + **Server Command**: + + ```shell Command + #One A3 card has 2 npu chips + sglang serve --tp-size 2 --sp-degree 1 --model-path Qwen/Qwen-Image --num-gpus 2 + ``` + + **Benchmark Command**: + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Qwen/Qwen-Image + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 36.26 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + Completed outputs: 1 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.03 + Output throughput (outputs/s): 0.03 + Latency Mean (s): 36.26 + Latency Median (s): 36.26 + Latency P90 (s): 36.26 + Latency P95 (s): 36.26 + Latency P99 (s): 36.26 + -------------------------------------------------- + Peak Memory Max (MB): 36984.00 + Peak Memory Mean (MB): 36984.00 + Peak Memory Median (MB): 36984.00 + ------------------------------------------------------------ + ``` + + #### 5.1.2 Generate images with high concurrency -**Benchmark Command**: + + + **Benchmark Command**: -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-image --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-image --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 --port 30000 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Task: text-to-image -Model: Qwen/Qwen-Image -Dataset: vbench --------------------------------------------------- -Benchmark duration (s): 300.79 -Request rate: inf -Max request concurrency: 20 -Successful requests: 14/20 --------------------------------------------------- -Request throughput (req/s): 0.05 -Latency Mean (s): 154.5368 -Latency Median (s): 154.8363 -Latency P99 (s): 285.4603 --------------------------------------------------- -Peak Memory Max (MB): 48030.31 -Peak Memory Mean (MB): 48030.30 -Peak Memory Median (MB): 48030.29 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Qwen/Qwen-Image + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 300.79 + Request rate: inf + Max request concurrency: 20 + Successful requests: 14/20 + -------------------------------------------------- + Request throughput (req/s): 0.05 + Latency Mean (s): 154.5368 + Latency Median (s): 154.8363 + Latency P99 (s): 285.4603 + -------------------------------------------------- + Peak Memory Max (MB): 48030.31 + Peak Memory Mean (MB): 48030.30 + Peak Memory Median (MB): 48030.29 + ============================================================ + ``` + + + **Benchmark Command**: + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Qwen/Qwen-Image + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 300.81 + Request rate: inf + Max request concurrency: 20 + Successful requests: 8/20 + Completed outputs: 8 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.03 + Output throughput (outputs/s): 0.03 + Latency Mean (s): 166.61 + Latency Median (s): 167.02 + Latency P90 (s): 270.80 + Latency P95 (s): 283.48 + Latency P99 (s): 293.64 + -------------------------------------------------- + Peak Memory Max (MB): 36984.00 + Peak Memory Mean (MB): 36984.00 + Peak Memory Median (MB): 36984.00 + ------------------------------------------------------------ + ``` + + diff --git a/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx b/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx index 479ff5b95..149689961 100644 --- a/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx +++ b/docs_new/cookbook/diffusion/Wan/Wan2.1.mdx @@ -35,7 +35,7 @@ This section provides deployment configurations optimized for different hardware ### 3.1 Basic Configuration -The Wan2.1 series offers models in multiple sizes and resolutions. SGLang supports Wan2.1 deployment on NVIDIA B200, B300, H200, H100, and AMD MI300X, MI325X, MI355X GPUs. The recommended launch configurations vary by hardware, model size, and memory headroom. +The Wan2.1 series offers models in multiple sizes and resolutions. SGLang supports Wan2.1 deployment on NVIDIA B200, B300, H200, H100, and AMD MI300X, MI325X, MI355X GPUs and Ascend A2, A3 NPUs. The recommended launch configurations vary by hardware, model size, and memory headroom. **Interactive Command Generator**: Use the configuration selector below to automatically generate an appropriate deployment command for your model variant and options. @@ -185,86 +185,207 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe #### 5.1.1 Generate a single video -**Server Command**: + + + **Server Command**: -```bash Command -sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers -``` + ```bash Command + sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers + ``` -**Benchmark Command**: + **Benchmark Command**: -```bash Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-video --dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1 -``` + ```bash Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-video --dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Task: text-to-video -Model: Wan-AI/Wan2.1-T2V-14B-Diffusers -Dataset: vbench --------------------------------------------------- -Benchmark duration (s): 1958.41 -Request rate: inf -Max request concurrency: 1 -Successful requests: 1/1 --------------------------------------------------- -Request throughput (req/s): 0.00 -Latency Mean (s): 1958.4059 -Latency Median (s): 1958.4059 -Latency P99 (s): 1958.4059 --------------------------------------------------- -Peak Memory Max (MB): 59662.00 -Peak Memory Mean (MB): 59662.00 -Peak Memory Median (MB): 59662.00 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-video + Model: Wan-AI/Wan2.1-T2V-14B-Diffusers + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 1958.41 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Latency Mean (s): 1958.4059 + Latency Median (s): 1958.4059 + Latency P99 (s): 1958.4059 + -------------------------------------------------- + Peak Memory Max (MB): 59662.00 + Peak Memory Mean (MB): 59662.00 + Peak Memory Median (MB): 59662.00 + ============================================================ + ``` + + + + **Server Command**: + + ```bash Command + #One A3 card has 2 npu chips. Benchmark was did with two A3 cards + sglang serve \ + --model-path /models/Wan-AI/Wan2.1-T2V-14B-Diffusers/ \ + --tp-size 2 \ + --sp-degree 2 \ + --num-gpus 4 \ + --attention-backend laser_attn + ``` + + **Benchmark Command**: + + ```bash Command + python -m sglang.multimodal_gen.benchmarks.bench_serving \ + --dataset vbench \ + --task text-to-video \ + --num-prompts 1 \ + --max-concurrency 1 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-video + Model: Wan-AI/Wan2.1-T2V-14B-Diffusers/ + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 1282.90 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + Completed outputs: 1 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Output throughput (outputs/s): 0.00 + Latency Mean (s): 1282.90 + Latency Median (s): 1282.90 + Latency P90 (s): 1282.90 + Latency P95 (s): 1282.90 + Latency P99 (s): 1282.90 + -------------------------------------------------- + Peak Memory Max (MB): 31938.00 + Peak Memory Mean (MB): 31938.00 + Peak Memory Median (MB): 31938.00 + ============================================================ + ``` + + #### 5.1.2 Generate videos with Cache-DiT acceleration -**Server Command**: + + + **Server Command**: -```bash Command -SGLANG_CACHE_DIT_ENABLED=true \ -SGLANG_CACHE_DIT_FN=2 \ -SGLANG_CACHE_DIT_BN=1 \ -SGLANG_CACHE_DIT_WARMUP=4 \ -SGLANG_CACHE_DIT_RDT=0.4 \ -SGLANG_CACHE_DIT_MC=4 \ -SGLANG_CACHE_DIT_TAYLORSEER=true \ -SGLANG_CACHE_DIT_TS_ORDER=2 \ -sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers -``` + ```bash Command + SGLANG_CACHE_DIT_ENABLED=true \ + SGLANG_CACHE_DIT_FN=2 \ + SGLANG_CACHE_DIT_BN=1 \ + SGLANG_CACHE_DIT_WARMUP=4 \ + SGLANG_CACHE_DIT_RDT=0.4 \ + SGLANG_CACHE_DIT_MC=4 \ + SGLANG_CACHE_DIT_TAYLORSEER=true \ + SGLANG_CACHE_DIT_TS_ORDER=2 \ + sglang serve --model-path Wan-AI/Wan2.1-T2V-14B-Diffusers + ``` -**Benchmark Command**: + **Benchmark Command**: -```bash Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-video --dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1 -``` + ```bash Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-video --dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Task: text-to-video -Model: Wan-AI/Wan2.1-T2V-14B-Diffusers -Dataset: vbench --------------------------------------------------- -Benchmark duration (s): 556.99 -Request rate: inf -Max request concurrency: 1 -Successful requests: 1/1 --------------------------------------------------- -Request throughput (req/s): 0.00 -Latency Mean (s): 556.9885 -Latency Median (s): 556.9885 -Latency P99 (s): 556.9885 --------------------------------------------------- -Peak Memory Max (MB): 69306.00 -Peak Memory Mean (MB): 69306.00 -Peak Memory Median (MB): 69306.00 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-video + Model: Wan-AI/Wan2.1-T2V-14B-Diffusers + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 556.99 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Latency Mean (s): 556.9885 + Latency Median (s): 556.9885 + Latency P99 (s): 556.9885 + -------------------------------------------------- + Peak Memory Max (MB): 69306.00 + Peak Memory Mean (MB): 69306.00 + Peak Memory Median (MB): 69306.00 + ============================================================ + ``` + + + + **Server Command**: + + ```bash Command + #One A3 card has 2 npu chips. Benchmark was did with two Atlas 3 cards + SGLANG_CACHE_DIT_FN=2 \ + SGLANG_CACHE_DIT_BN=1 \ + SGLANG_CACHE_DIT_WARMUP=4 \ + SGLANG_CACHE_DIT_RDT=0.4 \ + SGLANG_CACHE_DIT_MC=4 \ + SGLANG_CACHE_DIT_TAYLORSEER=true \ + SGLANG_CACHE_DIT_TS_ORDER=2 \ + SGLANG_CACHE_DIT_ENABLED=true sglang serve \ + --model-path /models/Wan-AI/Wan2.1-T2V-14B-Diffusers/ \ + --tp-size 2 \ + --sp-degree 2 \ + --num-gpus 4 \ + --attention-backend laser_attn + ``` + + **Benchmark Command**: + + ```bash Command + python -m sglang.multimodal_gen.benchmarks.bench_serving \ + --dataset vbench \ + --task text-to-video \ + --num-prompts 1 \ + --max-concurrency 1 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-video + Model: Wan-AI/Wan2.1-T2V-14B-Diffusers/ + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 413.88 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + Completed outputs: 1 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Output throughput (outputs/s): 0.00 + Latency Mean (s): 413.88 + Latency Median (s): 413.88 + Latency P90 (s): 413.88 + Latency P95 (s): 413.88 + Latency P99 (s): 413.88 + -------------------------------------------------- + Peak Memory Max (MB): 32782.00 + Peak Memory Mean (MB): 32782.00 + Peak Memory Median (MB): 32782.00 + ============================================================ + ``` + + diff --git a/docs_new/cookbook/diffusion/Wan/Wan2.2.mdx b/docs_new/cookbook/diffusion/Wan/Wan2.2.mdx index b87ca4475..256e98866 100644 --- a/docs_new/cookbook/diffusion/Wan/Wan2.2.mdx +++ b/docs_new/cookbook/diffusion/Wan/Wan2.2.mdx @@ -33,7 +33,7 @@ This section provides deployment configurations optimized for different hardware The Wan2.2 series offers models in various sizes, architectures and input types, optimized for different hardware platforms. The recommended launch configurations vary by hardware and model size. -**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size. SGLang supports serving Wan2.2 on NVIDIA B200, H200 and AMD MI300X, MI325X and MI355X GPUs. +**Interactive Command Generator**: Use the configuration selector below to automatically generate the appropriate deployment command for your hardware platform, model size. SGLang supports serving Wan2.2 on NVIDIA B200, H200, AMD MI300X, MI325X, MI355X GPUs and Ascend A2, A3 NPUs. @@ -65,7 +65,7 @@ curl http://127.0.0.1:3000/v1/images/generations \ -H "Content-Type: application/json" \ -H "Authorization: Bearer $OPENAI_API_KEY" \ -d '{ - "model": "black-forest-labs/FLUX.1-dev", + "model": "Wan-AI/Wan2.2-T2V-A14B-Diffusers", "prompt": "A cute baby sea otter", "n": 1, "size": "1024x1024", @@ -255,92 +255,207 @@ Test Environment: ### 5.1 Speedup Benchmark -#### 5.1.1 Generate a video +### 5.1.1 Generate a video -**Server Command**: + + + **Server Command**: + ```shell Command + sglang serve --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers + ``` -```shell Command -sglang serve --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers -``` + **Benchmark Command**: + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1 + ``` -**Benchmark Command**: + **Result**: + ```text Output + ================= Serving Benchmark Result ================= + Backend: sglang-video + Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers + Dataset: vbench + Task: t2v + -------------------------------------------------- + Benchmark duration (s): 630.43 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Latency Mean (s): 630.4277 + Latency Median (s): 630.4277 + Latency P99 (s): 630.4277 + -------------------------------------------------- + Peak Memory Max (MB): 62627.41 + Peak Memory Mean (MB): 62627.41 + Peak Memory Median (MB): 62627.41 -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1 -``` + ============================================================ + ``` + -**Result**: + + **Server Command**: + ```shell Command + #One A3 card has 2 npu chips. Using four A3 cards in benchmarking + sglang serve \ + --model-path /models/Wan-AI/Wan2.2-T2V-A14B-Diffusers/ \ + --tp-size 2 \ + --sp-degree 4 \ + --num-gpus 8 \ + --attention-backend laser_attn + ``` -```text Output -================= Serving Benchmark Result ================= -Backend: sglang-video -Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers -Dataset: vbench -Task: t2v --------------------------------------------------- -Benchmark duration (s): 630.43 -Request rate: inf -Max request concurrency: 1 -Successful requests: 1/1 --------------------------------------------------- -Request throughput (req/s): 0.00 -Latency Mean (s): 630.4277 -Latency Median (s): 630.4277 -Latency P99 (s): 630.4277 --------------------------------------------------- -Peak Memory Max (MB): 62627.41 -Peak Memory Mean (MB): 62627.41 -Peak Memory Median (MB): 62627.41 + **Benchmark Command**: + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving \ + --dataset vbench \ + --task text-to-video \ + --num-prompts 1 \ + --max-concurrency 1 + ``` -============================================================ -``` + **Result**: + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-video + Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers/ + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 214.50 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + Completed outputs: 1 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Output throughput (outputs/s): 0.00 + Latency Mean (s): 214.50 + Latency Median (s): 214.50 + Latency P90 (s): 214.50 + Latency P95 (s): 214.50 + Latency P99 (s): 214.50 + -------------------------------------------------- + Peak Memory Max (MB): 46692.00 + Peak Memory Mean (MB): 46692.00 + Peak Memory Median (MB): 46692.00 + ------------------------------------------------------------ + ``` + + #### 5.1.2 Generate videos with high concurrency -**Server Command**: + + + **Server Command**: -```shell Command -SGLANG_CACHE_DIT_ENABLED=true \ -SGLANG_CACHE_DIT_FN=2 \ -SGLANG_CACHE_DIT_BN=1 \ -SGLANG_CACHE_DIT_WARMUP=4 \ -SGLANG_CACHE_DIT_RDT=0.4 \ -SGLANG_CACHE_DIT_MC=4 \ -SGLANG_CACHE_DIT_TAYLORSEER=true \ -SGLANG_CACHE_DIT_TS_ORDER=2 \ -sglang serve --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers -``` + ```shell Command + SGLANG_CACHE_DIT_ENABLED=true \ + SGLANG_CACHE_DIT_FN=2 \ + SGLANG_CACHE_DIT_BN=1 \ + SGLANG_CACHE_DIT_WARMUP=4 \ + SGLANG_CACHE_DIT_RDT=0.4 \ + SGLANG_CACHE_DIT_MC=4 \ + SGLANG_CACHE_DIT_TAYLORSEER=true \ + SGLANG_CACHE_DIT_TS_ORDER=2 \ + sglang serve --model-path Wan-AI/Wan2.2-T2V-A14B-Diffusers + ``` -**Benchmark Command**: + **Benchmark Command**: -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-video --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-video --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Backend: sglang-video -Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers -Dataset: vbench -Task: t2v --------------------------------------------------- -Benchmark duration (s): 5163.21 -Request rate: inf -Max request concurrency: 20 -Successful requests: 20/20 --------------------------------------------------- -Request throughput (req/s): 0.00 -Latency Mean (s): 2739.7695 -Latency Median (s): 2742.0673 -Latency P99 (s): 5121.6331 --------------------------------------------------- -Peak Memory Max (MB): 72523.56 -Peak Memory Mean (MB): 70253.34 -Peak Memory Median (MB): 70824.46 + ```text Output + ================= Serving Benchmark Result ================= + Backend: sglang-video + Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers + Dataset: vbench + Task: t2v + -------------------------------------------------- + Benchmark duration (s): 5163.21 + Request rate: inf + Max request concurrency: 20 + Successful requests: 20/20 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Latency Mean (s): 2739.7695 + Latency Median (s): 2742.0673 + Latency P99 (s): 5121.6331 + -------------------------------------------------- + Peak Memory Max (MB): 72523.56 + Peak Memory Mean (MB): 70253.34 + Peak Memory Median (MB): 70824.46 -============================================================ -``` + ============================================================ + ``` + + + + **Server Command**: + + ```shell Command + #One A3 card has 2 npu chips. Using four A3 cards in benchmarking + SGLANG_CACHE_DIT_FN=2 \ + SGLANG_CACHE_DIT_BN=1 \ + SGLANG_CACHE_DIT_WARMUP=4 \ + SGLANG_CACHE_DIT_RDT=0.4 \ + SGLANG_CACHE_DIT_MC=4 \ + SGLANG_CACHE_DIT_TAYLORSEER=true \ + SGLANG_CACHE_DIT_TS_ORDER=2 \ + SGLANG_CACHE_DIT_ENABLED=true sglang serve \ + --model-path /models/Wan-AI/Wan2.2-T2V-A14B-Diffusers/ \ + --tp-size 2 \ + --sp-degree 4 \ + --num-gpus 8 \ + --attention-backend laser_attn + ``` + + **Benchmark Command**: + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving \ + --dataset vbench \ + --task text-to-video \ + --num-prompts 20 \ + --max-concurrency 20 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-video + Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers/ + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 4384.65 + Request rate: inf + Max request concurrency: 20 + Successful requests: 20/20 + Completed outputs: 20 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.00 + Output throughput (outputs/s): 0.00 + Latency Mean (s): 2304.17 + Latency Median (s): 2297.69 + Latency P90 (s): 3972.32 + Latency P95 (s): 4178.99 + Latency P99 (s): 4343.52 + -------------------------------------------------- + Peak Memory Max (MB): 46692.00 + Peak Memory Mean (MB): 46691.90 + Peak Memory Median (MB): 46692.00 + ------------------------------------------------------------ + ``` + + diff --git a/docs_new/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx b/docs_new/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx index e336c9330..fb5c29a29 100644 --- a/docs_new/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx +++ b/docs_new/cookbook/diffusion/Z-Image/Z-Image-Turbo.mdx @@ -209,73 +209,160 @@ Test Environment: #### 5.1.1 Generate an image -**Server Command**: + + + **Server Command**: -```shell Command -sglang serve --model-path Tongyi-MAI/Z-Image-Turbo \ - --ulysses-degree=1 --ring-degree=1 --port 30000 -``` + ```shell Command + sglang serve --model-path Tongyi-MAI/Z-Image-Turbo \ + --ulysses-degree=1 --ring-degree=1 --port 30000 + ``` -**Benchmark Command**: + **Benchmark Command**: -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-image --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-image --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Task: text-to-image -Model: Tongyi-MAI/Z-Image-Turbo -Dataset: vbench --------------------------------------------------- -Benchmark duration (s): 1.84 -Request rate: inf -Max request concurrency: 1 -Successful requests: 1/1 --------------------------------------------------- -Request throughput (req/s): 0.54 -Latency Mean (s): 1.8435 -Latency Median (s): 1.8435 -Latency P99 (s): 1.8435 --------------------------------------------------- -Peak Memory Max (MB): 30689.20 -Peak Memory Mean (MB): 30689.20 -Peak Memory Median (MB): 30689.20 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Tongyi-MAI/Z-Image-Turbo + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 1.84 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + -------------------------------------------------- + Request throughput (req/s): 0.54 + Latency Mean (s): 1.8435 + Latency Median (s): 1.8435 + Latency P99 (s): 1.8435 + -------------------------------------------------- + Peak Memory Max (MB): 30689.20 + Peak Memory Mean (MB): 30689.20 + Peak Memory Median (MB): 30689.20 + ============================================================ + ``` + + + **Server Command**: + + ```shell Command + #One A3 card has 2 npu chips + sglang serve --model-path Tongyi-MAI/Z-Image-Turbo --tp-size 2 --sp-degree 1 --num-gpus 2 + ``` + + **Benchmark Command**: + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Tongyi-MAI/Z-Image-Turbo + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 2.43 + Request rate: inf + Max request concurrency: 1 + Successful requests: 1/1 + Completed outputs: 1 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.41 + Output throughput (outputs/s): 0.41 + Latency Mean (s): 2.43 + Latency Median (s): 2.43 + Latency P90 (s): 2.43 + Latency P95 (s): 2.43 + Latency P99 (s): 2.43 + -------------------------------------------------- + Peak Memory Max (MB): 11052.00 + Peak Memory Mean (MB): 11052.00 + Peak Memory Median (MB): 11052.00 + ------------------------------------------------------------ + ``` + + #### 5.1.2 Generate images with high concurrency -**Benchmark Command**: + + + **Benchmark Command**: -```shell Command -python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ - --backend sglang-image --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 -``` + ```shell Command + python3 -m sglang.multimodal_gen.benchmarks.bench_serving \ + --backend sglang-image --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 + ``` -**Result**: + **Result**: -```text Output -================= Serving Benchmark Result ================= -Task: text-to-image -Model: Tongyi-MAI/Z-Image-Turbo -Dataset: vbench --------------------------------------------------- -Benchmark duration (s): 35.32 -Request rate: inf -Max request concurrency: 20 -Successful requests: 20/20 --------------------------------------------------- -Request throughput (req/s): 0.57 -Latency Mean (s): 18.5672 -Latency Median (s): 18.5573 -Latency P99 (s): 34.9880 --------------------------------------------------- -Peak Memory Max (MB): 30689.26 -Peak Memory Mean (MB): 30689.21 -Peak Memory Median (MB): 30689.21 -============================================================ -``` + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: Tongyi-MAI/Z-Image-Turbo + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 35.32 + Request rate: inf + Max request concurrency: 20 + Successful requests: 20/20 + -------------------------------------------------- + Request throughput (req/s): 0.57 + Latency Mean (s): 18.5672 + Latency Median (s): 18.5573 + Latency P99 (s): 34.9880 + -------------------------------------------------- + Peak Memory Max (MB): 30689.26 + Peak Memory Mean (MB): 30689.21 + Peak Memory Median (MB): 30689.21 + ============================================================ + ``` + + + **Benchmark Command**: + + ```shell Command + python -m sglang.multimodal_gen.benchmarks.bench_serving --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 + ``` + + **Result**: + + ```text Output + ================= Serving Benchmark Result ================= + Task: text-to-image + Model: /models/Tongyi-MAI/Z-Image-Turbo/Z-Image-Turbo + Dataset: vbench + -------------------------------------------------- + Benchmark duration (s): 49.08 + Request rate: inf + Max request concurrency: 20 + Successful requests: 20/20 + Completed outputs: 20 + Outputs per prompt: 1 + -------------------------------------------------- + Request throughput (req/s): 0.41 + Output throughput (outputs/s): 0.41 + Latency Mean (s): 25.78 + Latency Median (s): 25.77 + Latency P90 (s): 44.42 + Latency P95 (s): 46.75 + Latency P99 (s): 48.61 + -------------------------------------------------- + Peak Memory Max (MB): 11054.00 + Peak Memory Mean (MB): 11054.00 + Peak Memory Median (MB): 11054.00 + ------------------------------------------------------------ + ``` + + diff --git a/docs_new/src/snippets/diffusion/flux-deployment.jsx b/docs_new/src/snippets/diffusion/flux-deployment.jsx index f3a830215..400f3d2ce 100644 --- a/docs_new/src/snippets/diffusion/flux-deployment.jsx +++ b/docs_new/src/snippets/diffusion/flux-deployment.jsx @@ -14,6 +14,8 @@ export const FluxDeployment = () => { { id: 'mi355x', label: 'MI355X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, + { id: 'a2', label: 'A2', default: false }, + { id: 'a3', label: 'A3', default: false } ] }, version: { @@ -32,9 +34,30 @@ export const FluxDeployment = () => { }, generateCommand: function(values) { - const { version } = values; + const { hardware, version } = values; const config = this.modelConfigs[version]; + if (hardware === 'a2') { + if (version === 'flux1-dev') { + return `sglang serve \\ + --model-path ${config.repoId} \\ + --num-gpus 1`; + } + + return `sglang serve \\ + --model-path ${config.repoId} \\ + --tp-size 2 \\ + --num-gpus 2`; + } + + if (hardware === 'a3') { + return `#One A3 card has 2 npu chips +sglang serve \\ + --tp-size 2 \\ + --model-path ${config.repoId} \\ + --num-gpus 2`; + } + return `sglang serve \\ --model-path ${config.repoId} \\ --ulysses-degree=1 \\ @@ -108,6 +131,19 @@ export const FluxDeployment = () => { return () => observer.disconnect(); }, []); + useEffect(() => { + const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; + const targetTabName = isAscend ? 'Ascend A3' : 'NVIDIA B200'; + + const allTabs = document.querySelectorAll('button, [role="tab"]'); + allTabs.forEach((tab) => { + const text = tab.textContent.trim(); + if (text === targetTabName && tab.getAttribute('aria-selected') !== 'true') { + tab.click(); + } + }); + }, [values.hardware]); + const handleRadioChange = (optionName, value) => { setValues((prev) => ({ ...prev, [optionName]: value })); }; diff --git a/docs_new/src/snippets/diffusion/qwen-image-deployment.jsx b/docs_new/src/snippets/diffusion/qwen-image-deployment.jsx index 2dbf6c4b6..7b2b1d7cf 100644 --- a/docs_new/src/snippets/diffusion/qwen-image-deployment.jsx +++ b/docs_new/src/snippets/diffusion/qwen-image-deployment.jsx @@ -13,7 +13,9 @@ export const QwenImageDeployment = () => { { id: 'h100', label: 'H100', default: false }, { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, - { id: 'mi355x', label: 'MI355X', default: false } + { id: 'mi355x', label: 'MI355X', default: false }, + { id: 'a2', label: 'A2', default: false }, + { id: 'a3', label: 'A3', default: false } ] }, precision: { @@ -33,6 +35,21 @@ export const QwenImageDeployment = () => { }, generateCommand: function(values) { + if (values.hardware === 'a2') { + return `sglang serve \\ + --model-path Qwen/Qwen-Image \\ + --num-gpus 1`; + } + + if (values.hardware === 'a3') { + return `#One A3 card has 2 npu chips +sglang serve \\ + --model-path Qwen/Qwen-Image \\ + --tp-size 1 \\ + --sp-degree 2 \\ + --num-gpus 2`; + } + const isBlackwell = ['b200', 'b300'].includes(values.hardware); const isNvfp4 = values.precision === 'nvfp4' && isBlackwell; const modelPath = isNvfp4 @@ -112,6 +129,22 @@ export const QwenImageDeployment = () => { return () => observer.disconnect(); }, []); + useEffect(() => { + let targetTabName = 'AMD MI300X'; + if (values.hardware === 'a2') targetTabName = 'Ascend A2'; + if (values.hardware === 'a3') targetTabName = 'Ascend A3'; + + const allTabs = document.querySelectorAll('button, [role="tab"]'); + + allTabs.forEach((tab) => { + const text = tab.textContent.trim(); + + if (text === targetTabName && tab.getAttribute('aria-selected') !== 'true') { + tab.click(); + } + }); + }, [values.hardware]); + const handleRadioChange = (optionName, value) => { setValues((prev) => { const next = { ...prev, [optionName]: value }; diff --git a/docs_new/src/snippets/diffusion/wan21-deployment.jsx b/docs_new/src/snippets/diffusion/wan21-deployment.jsx index 6a752ce3e..8073aa4f4 100644 --- a/docs_new/src/snippets/diffusion/wan21-deployment.jsx +++ b/docs_new/src/snippets/diffusion/wan21-deployment.jsx @@ -47,6 +47,8 @@ export const Wan21Deployment = () => { { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, + { id: 'a2', label: 'A2', default: false }, + { id: 'a3', label: 'A3', default: false } ], }, task: { @@ -64,10 +66,10 @@ export const Wan21Deployment = () => { }, bestPractice: { name: 'bestPractice', - title: 'Sequence Parallelism', + title: 'Optimization', items: [ { id: 'off', label: 'Standard', default: true }, - { id: 'on', label: 'Best Practice (4 GPUs)', default: false }, + { id: 'on', label: 'Best Practice', default: false }, ], }, }; @@ -114,6 +116,22 @@ export const Wan21Deployment = () => { return () => observer.disconnect(); }, []); + useEffect(() => { + const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; + + const targetTabName = isAscend ? 'Ascend A3' : 'NVIDIA B200'; + + const allTabs = document.querySelectorAll('button, [role="tab"]'); + + allTabs.forEach((tab) => { + const text = tab.textContent.trim(); + + if (text === targetTabName && tab.getAttribute('aria-selected') !== 'true') { + tab.click(); + } + }); + }, [values.hardware]); + const handleRadioChange = (optionName, itemId) => { setValues((prev) => { let next = { ...prev, [optionName]: itemId }; @@ -154,7 +172,7 @@ export const Wan21Deployment = () => { }; const generateCommand = () => { - const { task, modelsize, selectedLoraPath, bestPractice } = values; + const { hardware, task, modelsize, selectedLoraPath, bestPractice } = values; const configKey = `${task}-${modelsize}`; const config = modelConfigs[configKey]; @@ -162,6 +180,43 @@ export const Wan21Deployment = () => { return '# Error: Invalid configuration'; } + if (hardware === 'a2' || hardware === 'a3') { + const comment = hardware === 'a3' + ? '#One A3 card has 2 npu chips\n' + : ''; + const isBestPractice = bestPractice === 'on'; + let command; + + if (task === 't2v' && modelsize === '1_3b' && hardware === 'a2' && !isBestPractice) { + command = `${comment}sglang serve \\ + --model-path ${config.repoId} \\ + --num-gpus 1`; + } else { + const tpSize = modelsize === '1_3b' ? (isBestPractice ? 4 : 1) : 2; + const spDegree = modelsize === '14b' && isBestPractice ? 4 : 1; + const numGpus = isBestPractice ? 8 : (hardware === 'a3' ? 2 : 4); + + command = `${comment}sglang serve \\ + --model-path ${config.repoId} \\ + --tp-size ${tpSize} \\ + --sp-degree ${spDegree} \\ + --num-gpus ${numGpus}`; + } + + if (isBestPractice) { + command += ` \\\n --attention-backend laser_attn`; + } + + if ( + selectedLoraPath === 'NIVEDAN/wan2.1-lora' || + selectedLoraPath === 'valiantcat/Wan2.1-Fight-LoRA' + ) { + command += ` \\\n --lora-path ${selectedLoraPath}`; + } + + return command; + } + let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`; if (bestPractice === 'on') { diff --git a/docs_new/src/snippets/diffusion/wan22-deployment.jsx b/docs_new/src/snippets/diffusion/wan22-deployment.jsx index ddda4ffc4..c854eefb5 100644 --- a/docs_new/src/snippets/diffusion/wan22-deployment.jsx +++ b/docs_new/src/snippets/diffusion/wan22-deployment.jsx @@ -11,6 +11,8 @@ { id: 'mi300x', label: 'MI300X', default: false }, { id: 'mi325x', label: 'MI325X', default: false }, { id: 'mi355x', label: 'MI355X', default: false }, + { id: 'a2', label: 'A2', default: false }, + { id: 'a3', label: 'A3', default: false } ], }, task: { @@ -32,10 +34,10 @@ }, bestPractice: { name: 'bestPractice', - title: 'Sequence Parallelism', + title: 'Optimization', items: [ { id: 'off', label: 'Standard', default: true }, - { id: 'on', label: 'Best Practice (4 GPUs)', default: false }, + { id: 'on', label: 'Best Practice', default: false }, ], }, }; @@ -85,6 +87,22 @@ return modelConfigs[configKey]?.supportedLoras || []; })(); + useEffect(() => { + const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; + + const targetTabName = isAscend ? 'Ascend A3' : 'NVIDIA B200'; + + const allTabs = document.querySelectorAll('button, [role="tab"]'); + + allTabs.forEach((tab) => { + const text = tab.textContent.trim(); + + if (text === targetTabName && tab.getAttribute('aria-selected') !== 'true') { + tab.click(); + } + }); + }, [values.hardware]); + const handleRadioChange = (optionName, itemId) => { setValues((prev) => { const next = { ...prev, [optionName]: itemId }; @@ -117,6 +135,56 @@ return '# Error: Invalid configuration'; } + + if (hardware === 'a2' || hardware === 'a3') { + const comment = hardware === 'a3' + ? '#One A3 card has 2 npu chips\n' + : ''; + const isBestPractice = bestPractice === 'on'; + let command; + + if (task === 'ti2v') { + if (isBestPractice) { + command = `${comment}sglang serve \\ + --model-path ${config.repoId} \\ + --sp-degree 8 \\ + --num-gpus 8`; + } else if (hardware === 'a2') { + command = `${comment}sglang serve \\ + --model-path ${config.repoId} \\ + --num-gpus 1`; + } else { + command = `${comment}sglang serve \\ + --model-path ${config.repoId} \\ + --tp-size 1 \\ + --sp-degree 2 \\ + --num-gpus 2`; + } + } else { + const spDegree = isBestPractice ? 4 : 1; + const numGpus = isBestPractice ? 8 : (hardware === 'a3' ? 2 : 4); + + command = `${comment}sglang serve \\ + --model-path ${config.repoId} \\ + --tp-size 2 \\ + --sp-degree ${spDegree} \\ + --num-gpus ${numGpus}`; + } + + if (isBestPractice) { + command += ` \\\n --attention-backend laser_attn`; + } + + if ( + selectedLoraPath === 'lightx2v/Wan2.2-Distill-Loras' || + selectedLoraPath === 'Cseti/wan2.2-14B-Arcane_Jinx-lora-v1' + ) { + command += ` \\\n --lora-path ${selectedLoraPath}`; + } + + return command; + } + let command = `sglang serve \\\n --model-path ${config.repoId} \\\n --dit-layerwise-offload true`; if (bestPractice === 'on') { if (hardware === 'b300') { diff --git a/docs_new/src/snippets/diffusion/zimage-turbo-deployment.jsx b/docs_new/src/snippets/diffusion/zimage-turbo-deployment.jsx index 71d9d80f8..e66748b15 100644 --- a/docs_new/src/snippets/diffusion/zimage-turbo-deployment.jsx +++ b/docs_new/src/snippets/diffusion/zimage-turbo-deployment.jsx @@ -12,12 +12,31 @@ export const ZImageTurboDeployment = () => { { id: 'mi355x', label: 'MI355X', default: false }, { id: 'b200', label: 'B200', default: true }, { id: 'h200', label: 'H200', default: false }, - { id: 'h100', label: 'H100', default: false } + { id: 'h100', label: 'H100', default: false }, + { id: 'a2', label: 'A2', default: false }, + { id: 'a3', label: 'A3', default: false } ] } }, generateCommand: function(values) { + const { hardware } = values; + + if (hardware === 'a2') { + return `sglang serve \\ + --model-path Tongyi-MAI/Z-Image-Turbo \\ + --num-gpus 1`; + } + + if (hardware === 'a3') { + return `#One A3 card has 2 npu chips +sglang serve \\ + --model-path Tongyi-MAI/Z-Image-Turbo \\ + --tp-size 2 \\ + --sp-degree 1 \\ + --num-gpus 2`; + } + return `sglang serve \\ --model-path Tongyi-MAI/Z-Image-Turbo \\ --ulysses-degree=1 \\ @@ -91,6 +110,19 @@ export const ZImageTurboDeployment = () => { return () => observer.disconnect(); }, []); + useEffect(() => { + const isAscend = values.hardware === 'a2' || values.hardware === 'a3'; + const targetTabName = isAscend ? 'Ascend A2 / A3' : 'AMD MI300X'; + + const allTabs = document.querySelectorAll('button, [role="tab"]'); + allTabs.forEach((tab) => { + const text = tab.textContent.trim(); + if (text === targetTabName && tab.getAttribute('aria-selected') !== 'true') { + tab.click(); + } + }); + }, [values.hardware]); + const handleRadioChange = (optionName, value) => { setValues((prev) => ({ ...prev, [optionName]: value })); };