[diffusion] chore: refresh docs, retire stale knobs, and fix nightly attribution (#34663)
This commit is contained in:
@@ -202,7 +202,7 @@ jobs:
|
|||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
ref: ${{ inputs.ref || github.ref }}
|
ref: ${{ inputs.ref || github.sha }}
|
||||||
|
|
||||||
- name: Install dependencies
|
- name: Install dependencies
|
||||||
run: |
|
run: |
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ metatags:
|
|||||||
description: "Benchmark diffusion model serving throughput and latency with SGLang - supports image and video generation with flexible configurations."
|
description: "Benchmark diffusion model serving throughput and latency with SGLang - supports image and video generation with flexible configurations."
|
||||||
---
|
---
|
||||||
|
|
||||||
`sglang.multimodal_gen.benchmarks.bench_serving` is a command-line tool designed to benchmark the online serving throughput and latency of Diffusion Models. It supports two backends (`sglang-image`, `sglang-video`) and offers flexible configurations for request rates, dataset types, and profiling.
|
`sglang.multimodal_gen.benchmarks.bench_serving` is a command-line tool designed to benchmark the online serving throughput and latency of diffusion models. It selects the image or video API from the requested task and offers flexible configurations for request rates, dataset types, and profiling.
|
||||||
|
|
||||||
## 1. Quick Start
|
## 1. Quick Start
|
||||||
|
|
||||||
@@ -15,23 +15,23 @@ Run a benchmark on a local server (port 30000) generating 1 videos/images from t
|
|||||||
```bash Command
|
```bash Command
|
||||||
# For text to video: such as Wan2.2-T2V-A14B-Diffusers
|
# For text to video: such as Wan2.2-T2V-A14B-Diffusers
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1
|
||||||
|
|
||||||
# For image to video: such as Wan2.2-I2V-A14B-Diffusers
|
# For image to video: such as Wan2.2-I2V-A14B-Diffusers
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task i2v --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task image-to-video --num-prompts 1 --max-concurrency 1
|
||||||
|
|
||||||
# For image-text to video: such as Wan2.2-TI2V-5B-Diffusers
|
# For image-text to video: such as Wan2.2-TI2V-5B-Diffusers
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task ti2v --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task image-to-video --num-prompts 1 --max-concurrency 1
|
||||||
|
|
||||||
# For text to image: such as Qwen-Image
|
# For text to image: such as Qwen-Image
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task t2i --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1
|
||||||
|
|
||||||
# For image-text to image: such as Qwen-Image-Edit
|
# For image-text to image: such as Qwen-Image-Edit
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task ti2i --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task image-to-image --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
### 1.2 Benchmarking in High Concurrency
|
### 1.2 Benchmarking in High Concurrency
|
||||||
@@ -41,28 +41,28 @@ Run a benchmark on a local server (port 30000) generating 20 videos/images from
|
|||||||
```bash Command
|
```bash Command
|
||||||
# For text to video: such as Wan2.2-T2V-A14B-Diffusers
|
# For text to video: such as Wan2.2-T2V-A14B-Diffusers
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task text-to-video --num-prompts 20 --max-concurrency 20
|
||||||
|
|
||||||
# For image to video: such as Wan2.2-I2V-A14B-Diffusers
|
# For image to video: such as Wan2.2-I2V-A14B-Diffusers
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task i2v --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task image-to-video --num-prompts 20 --max-concurrency 20
|
||||||
|
|
||||||
# For image-text to video: such as Wan2.2-TI2V-5B-Diffusers
|
# For image-text to video: such as Wan2.2-TI2V-5B-Diffusers
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task ti2v --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task image-to-video --num-prompts 20 --max-concurrency 20
|
||||||
|
|
||||||
# For text to image: such as Qwen-Image
|
# For text to image: such as Qwen-Image
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task t2i --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20
|
||||||
|
|
||||||
# For image-text to image: such as Qwen-Image-Edit
|
# For image-text to image: such as Qwen-Image-Edit
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task ti2i --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task image-to-image --num-prompts 20 --max-concurrency 20
|
||||||
```
|
```
|
||||||
|
|
||||||
## 2. Parameter Reference
|
## 2. Parameter Reference
|
||||||
|
|
||||||
### 2.1 Connection & Backend Settings
|
### 2.1 Connection Settings
|
||||||
|
|
||||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||||
<colgroup>
|
<colgroup>
|
||||||
@@ -78,11 +78,6 @@ python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
|||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--backend`</td>
|
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>**Required**</td>
|
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>The backend type to use. Choices: `sglang-image`, `sglang-video`.</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--base-url`</td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--base-url`</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`None`</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`None`</td>
|
||||||
@@ -124,8 +119,8 @@ python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
|||||||
<tbody>
|
<tbody>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--task`</td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--task`</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`t2v`, `i2v`, `ti2v`, `t2i`, `ti2i`</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>`text-to-video`, `image-to-video`, `text-to-image`, `image-to-image`, `video-to-video`</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Defines the generation task: `t2v` (Text-to-Video), `i2v` (Image-to-Video), `ti2v` (Text+Image-to-Video), `t2i` (Text-to-image), `ti2i` (Text+Image-to-Image).</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>Defines the generation task when it cannot be inferred from the model metadata.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--dataset`</td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--dataset`</td>
|
||||||
|
|||||||
@@ -230,17 +230,16 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|
||||||
```text Output
|
```text Output
|
||||||
================= Serving Benchmark Result =================
|
================= Serving Benchmark Result =================
|
||||||
Backend: sglang-image
|
|
||||||
Model: black-forest-labs/FLUX.1-dev
|
Model: black-forest-labs/FLUX.1-dev
|
||||||
Dataset: vbench
|
Dataset: vbench
|
||||||
Task: t2v
|
Task: text-to-image
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
Benchmark duration (s): 50.97
|
Benchmark duration (s): 50.97
|
||||||
Request rate: inf
|
Request rate: inf
|
||||||
@@ -317,17 +316,16 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result** :
|
**Result** :
|
||||||
|
|
||||||
```text Output
|
```text Output
|
||||||
================= Serving Benchmark Result =================
|
================= Serving Benchmark Result =================
|
||||||
Backend: sglang-image
|
|
||||||
Model: black-forest-labs/FLUX.1-dev
|
Model: black-forest-labs/FLUX.1-dev
|
||||||
Dataset: vbench
|
Dataset: vbench
|
||||||
Task: t2v
|
Task: text-to-image
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
Benchmark duration (s): 111.79
|
Benchmark duration (s): 111.79
|
||||||
Request rate: inf
|
Request rate: inf
|
||||||
|
|||||||
@@ -67,8 +67,6 @@ For two-stage pipelines, `--ltx2-two-stage-device-mode` controls transformer res
|
|||||||
| `resident` | Best latency on high-VRAM GPUs because both DiTs can stay resident. |
|
| `resident` | Best latency on high-VRAM GPUs because both DiTs can stay resident. |
|
||||||
| `original` | Closest to the original two-stage switching semantics. |
|
| `original` | Closest to the original two-stage switching semantics. |
|
||||||
|
|
||||||
`snapshot` is kept only as a deprecated compatibility alias for `original` and may be removed after two release cycles; use `original` or `resident` in new configs.
|
|
||||||
|
|
||||||
Other deployment flags:
|
Other deployment flags:
|
||||||
|
|
||||||
- `--lora-path`: Preload a community LoRA adapter.
|
- `--lora-path`: Preload a community LoRA adapter.
|
||||||
|
|||||||
@@ -0,0 +1,147 @@
|
|||||||
|
---
|
||||||
|
title: LingBot Video MoE
|
||||||
|
description: Serve the native LingBot Video MoE 30B-A3B text-to-video model with SGLang Diffusion.
|
||||||
|
metatags:
|
||||||
|
description: "Run robbyant/lingbot-video-moe-30b-a3b text-to-video generation with SGLang Diffusion."
|
||||||
|
---
|
||||||
|
|
||||||
|
import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx';
|
||||||
|
|
||||||
|
<DiffusionModelTags tags={["video", "text-to-video", "mixture-of-experts"]} />
|
||||||
|
|
||||||
|
## 1. Model introduction
|
||||||
|
|
||||||
|
[LingBot Video MoE 30B-A3B](https://huggingface.co/robbyant/lingbot-video-moe-30b-a3b)
|
||||||
|
is a text-to-video mixture-of-experts model. SGLang Diffusion provides a native
|
||||||
|
pipeline for the public checkpoint:
|
||||||
|
|
||||||
|
| Model ID | Task | Default output |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `robbyant/lingbot-video-moe-30b-a3b` | Text to video | 480x480, 81 frames at 16 FPS |
|
||||||
|
|
||||||
|
The checkpoint expects a structured JSON caption rather than an unexpanded
|
||||||
|
natural-language prompt. The JSON is passed as the request's `prompt` string;
|
||||||
|
it is not an `extra_params` object.
|
||||||
|
|
||||||
|
## 2. Installation
|
||||||
|
|
||||||
|
Install SGLang with the diffusion dependencies:
|
||||||
|
|
||||||
|
```bash Command
|
||||||
|
uv pip install "sglang[diffusion]" --prerelease=allow
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [SGLang Diffusion installation guide](/docs/sglang-diffusion/installation)
|
||||||
|
for platform-specific setup.
|
||||||
|
|
||||||
|
## 3. Serve LingBot Video MoE
|
||||||
|
|
||||||
|
Start the server with the Hugging Face model ID:
|
||||||
|
|
||||||
|
```bash Command
|
||||||
|
sglang serve \
|
||||||
|
--model-path robbyant/lingbot-video-moe-30b-a3b \
|
||||||
|
--port 30010
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. Generate a video
|
||||||
|
|
||||||
|
The following request uses the compact 17-frame, 12-step smoke-test profile.
|
||||||
|
Use the model defaults of 81 frames and 40 steps for the released generation
|
||||||
|
profile.
|
||||||
|
|
||||||
|
```python Python
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
base_url = "http://127.0.0.1:30010"
|
||||||
|
prompt = json.dumps(
|
||||||
|
{
|
||||||
|
"comprehensive_description": {
|
||||||
|
"scene_content_description": (
|
||||||
|
"A small silver robot arm on a white table slowly reaches "
|
||||||
|
"toward a red cube. The background is a softly lit laboratory wall."
|
||||||
|
),
|
||||||
|
"camera_movement_description": (
|
||||||
|
"The camera is static at eye level in a medium shot."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
"camera_info": {
|
||||||
|
"color": "Neutral",
|
||||||
|
"frame_size": "Medium",
|
||||||
|
"shot_type_angle": "Eye level",
|
||||||
|
"lens_size": "Medium",
|
||||||
|
"composition": "Center",
|
||||||
|
"lighting": "Soft light",
|
||||||
|
"lighting_type": "Artificial light",
|
||||||
|
},
|
||||||
|
"world_knowledge": [],
|
||||||
|
"prominent_elements": [
|
||||||
|
{
|
||||||
|
"name": "robot arm",
|
||||||
|
"description": "A small silver robot arm with a two-finger gripper.",
|
||||||
|
"actions": [
|
||||||
|
{
|
||||||
|
"timestamp": "[0.0s - 1.0s]",
|
||||||
|
"action": "reaches toward the red cube",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"location": "center of the frame",
|
||||||
|
"relative_size": "dominant",
|
||||||
|
"shape_and_color": "articulated silver metal arm",
|
||||||
|
"texture": "brushed metal",
|
||||||
|
"appearance_details": "two-finger gripper and visible joints",
|
||||||
|
"relationship": "reaching toward the red cube on the table",
|
||||||
|
"orientation": "upright, base on the table",
|
||||||
|
"pose": "reaching",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
separators=(",", ":"),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = requests.post(
|
||||||
|
f"{base_url}/v1/videos",
|
||||||
|
json={
|
||||||
|
"model": "robbyant/lingbot-video-moe-30b-a3b",
|
||||||
|
"prompt": prompt,
|
||||||
|
"size": "640x384",
|
||||||
|
"num_frames": 17,
|
||||||
|
"fps": 16,
|
||||||
|
"num_inference_steps": 12,
|
||||||
|
"guidance_scale": 6.0,
|
||||||
|
"flow_shift": 3.0,
|
||||||
|
"seed": 0,
|
||||||
|
},
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
video_id = response.json()["id"]
|
||||||
|
|
||||||
|
while True:
|
||||||
|
job = requests.get(f"{base_url}/v1/videos/{video_id}", timeout=30).json()
|
||||||
|
if job["status"] == "completed":
|
||||||
|
break
|
||||||
|
if job["status"] == "failed":
|
||||||
|
raise RuntimeError(job.get("error") or "Video generation failed")
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
video = requests.get(
|
||||||
|
f"{base_url}/v1/videos/{video_id}/content",
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
video.raise_for_status()
|
||||||
|
Path("lingbot_video_moe.mp4").write_bytes(video.content)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Request constraints
|
||||||
|
|
||||||
|
- `num_frames` must be `1` or `4n+1`; examples include 17 and 81.
|
||||||
|
- Width and height must both be multiples of 16.
|
||||||
|
- The native defaults are `guidance_scale=6.0`, `flow_shift=3.0`,
|
||||||
|
`num_inference_steps=40`, and `fps=16`.
|
||||||
|
- Keep the prompt as serialized JSON. Raw free text is outside the
|
||||||
|
checkpoint's expected caption format.
|
||||||
@@ -220,17 +220,16 @@ sglang serve --model-path Qwen/Qwen-Image-Edit-2511 --port 30000
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task ti2i --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task image-to-image --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|
||||||
```text Output
|
```text Output
|
||||||
================= Serving Benchmark Result =================
|
================= Serving Benchmark Result =================
|
||||||
Backend: sglang-image
|
|
||||||
Model: Qwen/Qwen-Image-Edit-2511
|
Model: Qwen/Qwen-Image-Edit-2511
|
||||||
Dataset: vbench
|
Dataset: vbench
|
||||||
Task: ti2i
|
Task: image-to-image
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
Benchmark duration (s): 35.31
|
Benchmark duration (s): 35.31
|
||||||
Request rate: inf
|
Request rate: inf
|
||||||
@@ -254,17 +253,16 @@ Peak Memory Median (MB): 47959.35
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task ti2i --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task image-to-image --num-prompts 20 --max-concurrency 20
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|
||||||
```text Output
|
```text Output
|
||||||
================= Serving Benchmark Result =================
|
================= Serving Benchmark Result =================
|
||||||
Backend: sglang-image
|
|
||||||
Model: Qwen/Qwen-Image-Edit-2511
|
Model: Qwen/Qwen-Image-Edit-2511
|
||||||
Dataset: vbench
|
Dataset: vbench
|
||||||
Task: ti2i
|
Task: image-to-image
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
Benchmark duration (s): 286.11
|
Benchmark duration (s): 286.11
|
||||||
Request rate: inf
|
Request rate: inf
|
||||||
|
|||||||
@@ -237,7 +237,7 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
@@ -317,7 +317,7 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 --port 30000
|
--dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20 --port 30000
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
---
|
||||||
|
title: SANA-Video
|
||||||
|
description: Serve the native SANA-Video 2B 480p text-to-video model with SGLang Diffusion.
|
||||||
|
metatags:
|
||||||
|
description: "Run Efficient-Large-Model/SANA-Video_2B_480p_diffusers text-to-video generation with SGLang Diffusion."
|
||||||
|
---
|
||||||
|
|
||||||
|
import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx';
|
||||||
|
|
||||||
|
<DiffusionModelTags tags={["video", "text-to-video"]} />
|
||||||
|
|
||||||
|
## 1. Model introduction
|
||||||
|
|
||||||
|
[SANA-Video 2B 480p](https://huggingface.co/Efficient-Large-Model/SANA-Video_2B_480p_diffusers)
|
||||||
|
is a text-to-video model with a native SGLang Diffusion pipeline.
|
||||||
|
|
||||||
|
| Model ID | Task | Default output |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` | Text to video | 832x480, 81 frames at 16 FPS |
|
||||||
|
|
||||||
|
## 2. Installation
|
||||||
|
|
||||||
|
Install SGLang with the diffusion dependencies:
|
||||||
|
|
||||||
|
```bash Command
|
||||||
|
uv pip install "sglang[diffusion]" --prerelease=allow
|
||||||
|
```
|
||||||
|
|
||||||
|
See the [SGLang Diffusion installation guide](/docs/sglang-diffusion/installation)
|
||||||
|
for platform-specific setup.
|
||||||
|
|
||||||
|
## 3. Serve SANA-Video
|
||||||
|
|
||||||
|
```bash Command
|
||||||
|
sglang serve \
|
||||||
|
--model-path Efficient-Large-Model/SANA-Video_2B_480p_diffusers \
|
||||||
|
--port 30010
|
||||||
|
```
|
||||||
|
|
||||||
|
## 4. Generate a video
|
||||||
|
|
||||||
|
The following request uses the compact 17-frame, 8-step profile covered by
|
||||||
|
server CI. Use the model defaults of 81 frames and 50 steps for the released
|
||||||
|
generation profile.
|
||||||
|
|
||||||
|
```python Python
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
base_url = "http://127.0.0.1:30010"
|
||||||
|
response = requests.post(
|
||||||
|
f"{base_url}/v1/videos",
|
||||||
|
json={
|
||||||
|
"model": "Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
||||||
|
"prompt": (
|
||||||
|
"A red tram moves slowly through a sunlit city square while "
|
||||||
|
"pedestrians cross behind it. motion score: 30."
|
||||||
|
),
|
||||||
|
"size": "832x480",
|
||||||
|
"num_frames": 17,
|
||||||
|
"fps": 16,
|
||||||
|
"num_inference_steps": 8,
|
||||||
|
"guidance_scale": 6.0,
|
||||||
|
"seed": 42,
|
||||||
|
},
|
||||||
|
timeout=60,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
video_id = response.json()["id"]
|
||||||
|
|
||||||
|
while True:
|
||||||
|
job = requests.get(f"{base_url}/v1/videos/{video_id}", timeout=30).json()
|
||||||
|
if job["status"] == "completed":
|
||||||
|
break
|
||||||
|
if job["status"] == "failed":
|
||||||
|
raise RuntimeError(job.get("error") or "Video generation failed")
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
video = requests.get(
|
||||||
|
f"{base_url}/v1/videos/{video_id}/content",
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
video.raise_for_status()
|
||||||
|
Path("sana_video.mp4").write_bytes(video.content)
|
||||||
|
```
|
||||||
|
|
||||||
|
## 5. Request constraints
|
||||||
|
|
||||||
|
- The default profile uses `832x480`, 81 frames, 50 inference steps, and 16 FPS.
|
||||||
|
- Frame counts are aligned to `4n+1`; for example, a request for 80 frames is
|
||||||
|
adjusted to 77.
|
||||||
|
- Use width and height values divisible by 16.
|
||||||
|
- The prompt supports an optional `motion score: N.` suffix to express the
|
||||||
|
desired amount of motion.
|
||||||
@@ -200,7 +200,7 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe
|
|||||||
|
|
||||||
```bash Command
|
```bash Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
@@ -304,7 +304,7 @@ You can use the built-in SGLang diffusion benchmark script to evaluate Wan2.1 pe
|
|||||||
|
|
||||||
```bash Command
|
```bash Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|||||||
@@ -270,16 +270,15 @@ Test Environment:
|
|||||||
**Benchmark Command**:
|
**Benchmark Command**:
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task t2v --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-video --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
```text Output
|
```text Output
|
||||||
================= Serving Benchmark Result =================
|
================= Serving Benchmark Result =================
|
||||||
Backend: sglang-video
|
|
||||||
Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
|
Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
|
||||||
Dataset: vbench
|
Dataset: vbench
|
||||||
Task: t2v
|
Task: text-to-video
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
Benchmark duration (s): 630.43
|
Benchmark duration (s): 630.43
|
||||||
Request rate: inf
|
Request rate: inf
|
||||||
@@ -372,17 +371,16 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-video --dataset vbench --task t2v --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task text-to-video --num-prompts 20 --max-concurrency 20
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|
||||||
```text Output
|
```text Output
|
||||||
================= Serving Benchmark Result =================
|
================= Serving Benchmark Result =================
|
||||||
Backend: sglang-video
|
|
||||||
Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
|
Model: Wan-AI/Wan2.2-T2V-A14B-Diffusers
|
||||||
Dataset: vbench
|
Dataset: vbench
|
||||||
Task: t2v
|
Task: text-to-video
|
||||||
--------------------------------------------------
|
--------------------------------------------------
|
||||||
Benchmark duration (s): 5163.21
|
Benchmark duration (s): 5163.21
|
||||||
Request rate: inf
|
Request rate: inf
|
||||||
|
|||||||
@@ -225,7 +225,7 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1
|
--dataset vbench --task text-to-image --num-prompts 1 --max-concurrency 1
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
@@ -305,7 +305,7 @@ Test Environment:
|
|||||||
|
|
||||||
```shell Command
|
```shell Command
|
||||||
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
python3 -m sglang.multimodal_gen.benchmarks.bench_serving \
|
||||||
--backend sglang-image --dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20
|
--dataset vbench --task text-to-image --num-prompts 20 --max-concurrency 20
|
||||||
```
|
```
|
||||||
|
|
||||||
**Result**:
|
**Result**:
|
||||||
|
|||||||
@@ -80,6 +80,12 @@ Video models denoise a bounded latent video sequence for each request. Use these
|
|||||||
href="/cookbook/diffusion/LTX/LTX2 & LTX2.3"
|
href="/cookbook/diffusion/LTX/LTX2 & LTX2.3"
|
||||||
img="/cards/logos/ltx.svg"
|
img="/cards/logos/ltx.svg"
|
||||||
/>
|
/>
|
||||||
|
<Card
|
||||||
|
title="SANA-Video"
|
||||||
|
mode="card"
|
||||||
|
href="/cookbook/diffusion/SANA-Video/SANA-Video"
|
||||||
|
img="/cards/logos/sana.png"
|
||||||
|
/>
|
||||||
<Card
|
<Card
|
||||||
title="JoyAI-Echo"
|
title="JoyAI-Echo"
|
||||||
mode="card"
|
mode="card"
|
||||||
@@ -98,6 +104,12 @@ Video models denoise a bounded latent video sequence for each request. Use these
|
|||||||
href="/cookbook/diffusion/MiniMax/MiniMax-H3"
|
href="/cookbook/diffusion/MiniMax/MiniMax-H3"
|
||||||
img="/cards/logos/minimax.png"
|
img="/cards/logos/minimax.png"
|
||||||
/>
|
/>
|
||||||
|
<Card
|
||||||
|
title="LingBot Video MoE"
|
||||||
|
mode="card"
|
||||||
|
href="/cookbook/diffusion/LingBot-Video/LingBot-Video-MoE"
|
||||||
|
img="/cards/logos/inclusionai.png"
|
||||||
|
/>
|
||||||
</CardGroup>
|
</CardGroup>
|
||||||
|
|
||||||
## Realtime / World Models
|
## Realtime / World Models
|
||||||
|
|||||||
@@ -1512,6 +1512,13 @@
|
|||||||
"cookbook/diffusion/MOVA/MOVA"
|
"cookbook/diffusion/MOVA/MOVA"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"group": "SANA Video",
|
||||||
|
"tag": "NEW",
|
||||||
|
"pages": [
|
||||||
|
"cookbook/diffusion/SANA-Video/SANA-Video"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"group": "MiniMax",
|
"group": "MiniMax",
|
||||||
"tag": "NEW",
|
"tag": "NEW",
|
||||||
@@ -1519,6 +1526,13 @@
|
|||||||
"cookbook/diffusion/MiniMax/MiniMax-H3"
|
"cookbook/diffusion/MiniMax/MiniMax-H3"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"group": "LingBot Video",
|
||||||
|
"tag": "NEW",
|
||||||
|
"pages": [
|
||||||
|
"cookbook/diffusion/LingBot-Video/LingBot-Video-MoE"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"group": "LingBot World",
|
"group": "LingBot World",
|
||||||
"pages": [
|
"pages": [
|
||||||
|
|||||||
@@ -89,6 +89,12 @@ Rows are grouped when a family shares the same runtime path or optimization supp
|
|||||||
<td>T2V, 480p</td>
|
<td>T2V, 480p</td>
|
||||||
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
|
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>LingBot Video MoE</td>
|
||||||
|
<td><div className="sgd-id-list"><code>robbyant/lingbot-video-moe-30b-a3b</code></div></td>
|
||||||
|
<td>T2V, 480p</td>
|
||||||
|
<td><span className="sgd-muted">No dedicated optimization listed</span></td>
|
||||||
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Wan2.2</td>
|
<td>Wan2.2</td>
|
||||||
<td><div className="sgd-id-list"><code>Wan-AI/Wan2.2-TI2V-5B-Diffusers</code><code>Wan-AI/Wan2.2-T2V-A14B-Diffusers</code><code>nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4</code><code>Wan-AI/Wan2.2-I2V-A14B-Diffusers</code></div></td>
|
<td><div className="sgd-id-list"><code>Wan-AI/Wan2.2-TI2V-5B-Diffusers</code><code>Wan-AI/Wan2.2-T2V-A14B-Diffusers</code><code>nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4</code><code>Wan-AI/Wan2.2-I2V-A14B-Diffusers</code></div></td>
|
||||||
@@ -285,6 +291,21 @@ Optimization columns are abbreviated to keep the matrix readable:
|
|||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>LingBot Video MoE 30B-A3B</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>robbyant/lingbot-video-moe-30b-a3b</code></td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>480p</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>❌</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>❌</td>
|
||||||
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FastWan2.2 TI2V 5B</td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>FastWan2.2 TI2V 5B</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers</code><br /><code>FastVideo/FastWan2.2-TI2V-5B-Diffusers</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}><code>FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers</code><br /><code>FastVideo/FastWan2.2-TI2V-5B-Diffusers</code></td>
|
||||||
@@ -678,7 +699,6 @@ Optimization columns are abbreviated to keep the matrix readable:
|
|||||||
- `original` keeps official two-stage semantics without the premerged stage-2 transformer path.
|
- `original` keeps official two-stage semantics without the premerged stage-2 transformer path.
|
||||||
- `resident` usually provides the best latency/throughput but uses much more VRAM.
|
- `resident` usually provides the best latency/throughput but uses much more VRAM.
|
||||||
- Default is auto: `resident` on H200/high-memory CUDA GPUs, otherwise `original`.
|
- Default is auto: `resident` on H200/high-memory CUDA GPUs, otherwise `original`.
|
||||||
- Deprecated compatibility: `snapshot` is accepted as an alias for `original` and may be removed after two release cycles.
|
|
||||||
5. Cosmos3 ships in two sizes — `nvidia/Cosmos3-Nano` (16B) and
|
5. Cosmos3 ships in two sizes — `nvidia/Cosmos3-Nano` (16B) and
|
||||||
`nvidia/Cosmos3-Super` (64B). Both share the same pipeline; the only
|
`nvidia/Cosmos3-Super` (64B). Both share the same pipeline; the only
|
||||||
difference is transformer depth and width, picked up from
|
difference is transformer depth and width, picked up from
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ SGLang diffusion features an end-to-end unified pipeline for accelerating diffus
|
|||||||
## Key Features
|
## Key Features
|
||||||
|
|
||||||
SGLang Diffusion has the following features:
|
SGLang Diffusion has the following features:
|
||||||
- Broad model support: Wan, FastWan, FLUX, Qwen-Image, Z-Image, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3, MiniMax-H3, LingBot World, SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
- Broad model support: Wan, FastWan, FLUX, Qwen-Image, Z-Image, Ideogram 4, Krea-2, Cosmos3, LTX-2/LTX-2.3, MiniMax-H3, LingBot Video MoE, LingBot World, SANA-Video/SANA-WM, JoyEcho, MOVA, GLM-Image, ERNIE-Image, Hunyuan3D, and more
|
||||||
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
|
- Fast inference speed: empowered by optimized `sgl-kernel` kernels, scheduler/runtime improvements, caching acceleration, and native diffusion hot-path optimizations
|
||||||
- Ease of use: OpenAI-compatible api, CLI, and python sdk support
|
- Ease of use: OpenAI-compatible api, CLI, and python sdk support
|
||||||
- Multi-platform support:
|
- Multi-platform support:
|
||||||
|
|||||||
@@ -728,12 +728,6 @@ if __name__ == "__main__":
|
|||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Benchmark serving for diffusion models."
|
description="Benchmark serving for diffusion models."
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--backend",
|
|
||||||
type=str,
|
|
||||||
default=None,
|
|
||||||
help="DEPRECATED: --task is deprecated and will be ignored. The task will be inferred from --model.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--base-url",
|
"--base-url",
|
||||||
type=str,
|
type=str,
|
||||||
|
|||||||
@@ -639,11 +639,12 @@ def _try_save_cuda_video_direct(
|
|||||||
stderr=stderr_file.read(),
|
stderr=stderr_file.read(),
|
||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.warning(
|
logger.warning_once(
|
||||||
"Direct CUDA video save failed; falling back to imageio: %s",
|
"Direct CUDA video save failed; falling back to imageio. "
|
||||||
str(e),
|
"Enable debug logging for exception details."
|
||||||
)
|
)
|
||||||
|
logger.debug("Direct CUDA video save failure", exc_info=True)
|
||||||
return False
|
return False
|
||||||
finally:
|
finally:
|
||||||
if tmp_wav_path:
|
if tmp_wav_path:
|
||||||
@@ -725,11 +726,12 @@ def _try_save_cuda_videos_direct(
|
|||||||
try:
|
try:
|
||||||
with ThreadPoolExecutor(max_workers=_MAX_PARALLEL_CUDA_VIDEO_SAVES) as pool:
|
with ThreadPoolExecutor(max_workers=_MAX_PARALLEL_CUDA_VIDEO_SAVES) as pool:
|
||||||
return list(pool.map(save_one, range(len(samples))))
|
return list(pool.map(save_one, range(len(samples))))
|
||||||
except Exception as exc:
|
except Exception:
|
||||||
logger.warning(
|
logger.warning_once(
|
||||||
"Parallel CUDA video save failed; falling back to serial output: %s",
|
"Parallel CUDA video save failed; falling back to serial output. "
|
||||||
str(exc),
|
"Enable debug logging for exception details."
|
||||||
)
|
)
|
||||||
|
logger.debug("Parallel CUDA video save failure", exc_info=True)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -151,8 +151,7 @@ class _SageAttentionBackendResolver(_CudaAttentionBackendResolver):
|
|||||||
def resolve(cls, platform) -> str | AttentionBackendEnum:
|
def resolve(cls, platform) -> str | AttentionBackendEnum:
|
||||||
try:
|
try:
|
||||||
from sageattention import sageattn # noqa: F401
|
from sageattention import sageattn # noqa: F401
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
logger.info(e)
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Sage Attention backend is not installed (To install it, run `pip install git+https://github.com/thu-ml/SageAttention.git@d9704247a5139ab4c03bf7fc6b35cc0e2cbb5ea4 --no-build-isolation`). Falling back to Flash Attention."
|
"Sage Attention backend is not installed (To install it, run `pip install git+https://github.com/thu-ml/SageAttention.git@d9704247a5139ab4c03bf7fc6b35cc0e2cbb5ea4 --no-build-isolation`). Falling back to Flash Attention."
|
||||||
)
|
)
|
||||||
@@ -176,8 +175,7 @@ class _SageAttentionBackendResolver(_CudaAttentionBackendResolver):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return "sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn.SageAttentionBackend"
|
return "sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn.SageAttentionBackend"
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
logger.info(e)
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Sage Attention backend failed to import. Falling back to Flash Attention."
|
"Sage Attention backend failed to import. Falling back to Flash Attention."
|
||||||
)
|
)
|
||||||
@@ -195,8 +193,7 @@ class _SageAttention3BackendResolver(_CudaAttentionBackendResolver):
|
|||||||
)
|
)
|
||||||
|
|
||||||
return "sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn3.SageAttention3Backend"
|
return "sglang.multimodal_gen.runtime.layers.attention.backends.sage_attn3.SageAttention3Backend"
|
||||||
except ImportError as e:
|
except ImportError:
|
||||||
logger.info(e)
|
|
||||||
logger.info(
|
logger.info(
|
||||||
"Sage Attention 3 backend is not installed (To install it, see https://github.com/thu-ml/SageAttention/tree/main/sageattention3_blackwell#installation). Falling back to Torch SDPA."
|
"Sage Attention 3 backend is not installed (To install it, see https://github.com/thu-ml/SageAttention/tree/main/sageattention3_blackwell#installation). Falling back to Torch SDPA."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ from sglang.multimodal_gen.utils import (
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
LTX2_TWO_STAGE_DEVICE_MODES = ("original", "resident")
|
LTX2_TWO_STAGE_DEVICE_MODES = ("original", "resident")
|
||||||
LTX2_TWO_STAGE_DEVICE_MODE_CHOICES = (*LTX2_TWO_STAGE_DEVICE_MODES, "snapshot")
|
LTX2_TWO_STAGE_DEVICE_MODE_CHOICES = LTX2_TWO_STAGE_DEVICE_MODES
|
||||||
LTX2_TWO_STAGE_PIPELINE_NAMES = ("LTX2TwoStagePipeline", "LTX2TwoStageHQPipeline")
|
LTX2_TWO_STAGE_PIPELINE_NAMES = ("LTX2TwoStagePipeline", "LTX2TwoStageHQPipeline")
|
||||||
# H200-class GPUs (>=130 GiB total) can usually keep both LTX2 DiTs resident.
|
# H200-class GPUs (>=130 GiB total) can usually keep both LTX2 DiTs resident.
|
||||||
LTX2_RESIDENT_AUTO_ENABLE_MEM_GB = 130
|
LTX2_RESIDENT_AUTO_ENABLE_MEM_GB = 130
|
||||||
@@ -100,15 +100,7 @@ RING_CAPABLE_ATTENTION_BACKENDS = ("fa", "sage_attn")
|
|||||||
def _normalize_ltx2_two_stage_device_mode(mode: str | None) -> str | None:
|
def _normalize_ltx2_two_stage_device_mode(mode: str | None) -> str | None:
|
||||||
if mode is None:
|
if mode is None:
|
||||||
return None
|
return None
|
||||||
mode = mode.lower()
|
return mode.lower()
|
||||||
if mode == "snapshot":
|
|
||||||
logger.warning(
|
|
||||||
"ltx2_two_stage_device_mode=snapshot is deprecated and is treated "
|
|
||||||
"as original. Please use ltx2_two_stage_device_mode=original or "
|
|
||||||
"resident instead. This alias may be removed after two release cycles."
|
|
||||||
)
|
|
||||||
return "original"
|
|
||||||
return mode
|
|
||||||
|
|
||||||
|
|
||||||
def is_ltx2_two_stage_pipeline_name(pipeline_class_name: str | None) -> bool:
|
def is_ltx2_two_stage_pipeline_name(pipeline_class_name: str | None) -> bool:
|
||||||
@@ -2211,8 +2203,6 @@ class ServerArgs(DisaggServerArgsMixin):
|
|||||||
"LTX-2.3 two-stage device residency mode: "
|
"LTX-2.3 two-stage device residency mode: "
|
||||||
"'original' keeps official two-stage semantics without premerged stage2, "
|
"'original' keeps official two-stage semantics without premerged stage2, "
|
||||||
"'resident' keeps both transformers resident on GPU. "
|
"'resident' keeps both transformers resident on GPU. "
|
||||||
"'snapshot' is deprecated, treated as 'original', and may be "
|
|
||||||
"removed after two release cycles. "
|
|
||||||
"Default is auto: resident on H200/high-memory CUDA GPUs, otherwise original."
|
"Default is auto: resident on H200/high-memory CUDA GPUs, otherwise original."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -2001,8 +2001,9 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
["text_encoder", "image_encoder", "vae"],
|
["text_encoder", "image_encoder", "vae"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_ltx23_snapshot_device_mode_is_deprecated_alias_for_original(self):
|
def test_ltx23_snapshot_device_mode_is_rejected(self):
|
||||||
args = self._from_dict_with_pipeline_config(
|
with self.assertRaisesRegex(ValueError, "Expected one of"):
|
||||||
|
self._from_dict_with_pipeline_config(
|
||||||
LTX2PipelineConfig(),
|
LTX2PipelineConfig(),
|
||||||
memory_gb=140,
|
memory_gb=140,
|
||||||
available_memory_gb=134,
|
available_memory_gb=134,
|
||||||
@@ -2014,12 +2015,6 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(args.ltx2_two_stage_device_mode, "original")
|
|
||||||
self.assertEqual(
|
|
||||||
args.layerwise_offload_components,
|
|
||||||
["text_encoder", "image_encoder", "vae"],
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_explicit_layerwise_components_preserved_in_ltx23_resident(self):
|
def test_explicit_layerwise_components_preserved_in_ltx23_resident(self):
|
||||||
args = self._from_dict_with_pipeline_config(
|
args = self._from_dict_with_pipeline_config(
|
||||||
LTX2PipelineConfig(),
|
LTX2PipelineConfig(),
|
||||||
@@ -2425,7 +2420,7 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
self.assertFalse(server_args.use_fsdp_inference)
|
self.assertFalse(server_args.use_fsdp_inference)
|
||||||
self.assertFalse(server_args.enable_cfg_parallel)
|
self.assertFalse(server_args.enable_cfg_parallel)
|
||||||
|
|
||||||
def test_ltx23_snapshot_device_mode_cli_alias_is_accepted(self):
|
def test_ltx23_snapshot_device_mode_cli_is_rejected(self):
|
||||||
parser = FlexibleArgumentParser()
|
parser = FlexibleArgumentParser()
|
||||||
ServerArgs.add_cli_args(parser)
|
ServerArgs.add_cli_args(parser)
|
||||||
argv = [
|
argv = [
|
||||||
@@ -2437,36 +2432,8 @@ class TestOffloadDefaults(unittest.TestCase):
|
|||||||
"snapshot",
|
"snapshot",
|
||||||
]
|
]
|
||||||
|
|
||||||
with (
|
with self.assertRaises(SystemExit):
|
||||||
patch.object(sys, "argv", ["sglang"] + argv),
|
parser.parse_known_args(argv)
|
||||||
patch.object(
|
|
||||||
PipelineConfig, "from_kwargs", return_value=LTX2PipelineConfig()
|
|
||||||
),
|
|
||||||
patch(
|
|
||||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
|
|
||||||
return_value=False,
|
|
||||||
),
|
|
||||||
patch(
|
|
||||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_mps",
|
|
||||||
return_value=False,
|
|
||||||
),
|
|
||||||
patch(
|
|
||||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
|
|
||||||
return_value=True,
|
|
||||||
),
|
|
||||||
patch(
|
|
||||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_total_memory",
|
|
||||||
return_value=140 * 1024**3,
|
|
||||||
),
|
|
||||||
patch(
|
|
||||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_available_gpu_memory",
|
|
||||||
return_value=134,
|
|
||||||
),
|
|
||||||
):
|
|
||||||
args, unknown_args = parser.parse_known_args(argv)
|
|
||||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
|
||||||
|
|
||||||
self.assertEqual(server_args.ltx2_two_stage_device_mode, "original")
|
|
||||||
|
|
||||||
|
|
||||||
class TestKVGatherDegree(unittest.TestCase):
|
class TestKVGatherDegree(unittest.TestCase):
|
||||||
|
|||||||
@@ -868,6 +868,20 @@ def _install_framework(fw_name: str, dry_run: bool = False) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _get_checkout_commit_sha() -> str:
|
||||||
|
fallback = os.environ.get("GITHUB_SHA", "unknown")
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "rev-parse", "HEAD"],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.CalledProcessError):
|
||||||
|
return fallback
|
||||||
|
return result.stdout.strip() or fallback
|
||||||
|
|
||||||
|
|
||||||
def run_comparison(
|
def run_comparison(
|
||||||
config: dict,
|
config: dict,
|
||||||
case_ids: list[str] | None = None,
|
case_ids: list[str] | None = None,
|
||||||
@@ -882,7 +896,7 @@ def run_comparison(
|
|||||||
Each non-sglang framework is installed right before its cases run.
|
Each non-sglang framework is installed right before its cases run.
|
||||||
"""
|
"""
|
||||||
timestamp = datetime.now(timezone.utc).isoformat()
|
timestamp = datetime.now(timezone.utc).isoformat()
|
||||||
commit_sha = os.environ.get("GITHUB_SHA", "unknown")
|
commit_sha = _get_checkout_commit_sha()
|
||||||
run_id = os.environ.get("GITHUB_RUN_ID", "local")
|
run_id = os.environ.get("GITHUB_RUN_ID", "local")
|
||||||
|
|
||||||
log_dir = Path("comparison-logs")
|
log_dir = Path("comparison-logs")
|
||||||
|
|||||||
Reference in New Issue
Block a user