minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

diffusion: fix wan2.2 ti2v num_frames adjust logic (#13379)

Browse Source 
Co-authored-by: adarshxs <adarsh.shirawalmath@gmail.com>
Co-authored-by: Adarsh Shirawalmath <114558126+adarshxs@users.noreply.github.com>
This commit is contained in:
Mick
2025-11-17 20:53:15 +08:00
committed by GitHub
co-authored by adarshxs Adarsh Shirawalmath
parent ac406d4301
commit 7afff8fd1a
11 changed files with 211 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/pr-test.yml
+3 -2
View File
@@ -322,9 +322,11 @@ jobs:
- name: Clean Corrupted Hugging Face Model Cache - name: Clean Corrupted Hugging Face Model Cache
run: | run: |
echo "Temp: Deleting potentially corrupted Qwen/Qwen-Image and Qwen/Qwen-Image-Edit cache to ensure a fresh download." echo "Temp: Deleting potentially corrupted Qwen/Qwen-Image and Qwen/Qwen-Image-Edit cache to ensure a fresh download. This is temporary"
rm -rf /hf_home/hub/models--Qwen--Qwen-Image rm -rf /hf_home/hub/models--Qwen--Qwen-Image
rm -rf /hf_home/hub/models--Qwen--Qwen-Image-Edit rm -rf /hf_home/hub/models--Qwen--Qwen-Image-Edit
rm -rf /hf_home/hub/models--Wan-AI--Wan2.2-I2V-A14B-Diffusers
rm -rf /hf_home/hub/models--Wan-AI--Wan2.2-TI2V-5B-Diffusers
- name: Run diffusion server tests - name: Run diffusion server tests
timeout-minutes: 60 timeout-minutes: 60
@@ -1009,7 +1011,6 @@ jobs:
exit 1 exit 1
fi fi
done done
# If the loop completes, all jobs were successful # If the loop completes, all jobs were successful
echo "All jobs completed successfully" echo "All jobs completed successfully"
exit 0 exit 0
python/sglang/multimodal_gen/configs/pipelines/base.py
+5 -2
View File
@@ -37,7 +37,7 @@ class ModelTaskType(Enum):
T2I = auto() # Text to Image T2I = auto() # Text to Image
I2I = auto() # Image to Image I2I = auto() # Image to Image
def is_image_task(self): def is_image_gen(self):
return self == ModelTaskType.T2I or self == ModelTaskType.I2I return self == ModelTaskType.T2I or self == ModelTaskType.I2I
@@ -134,12 +134,15 @@ class PipelineConfig:
def slice_noise_pred(self, noise, latents): def slice_noise_pred(self, noise, latents):
return noise return noise
def set_width_and_height(self, width, height, image): def adjust_size(self, width, height, image):
""" """
image: input image image: input image
""" """
return width, height return width, height
def adjust_num_frames(self, num_frames):
return num_frames
# called in ImageEncodingStage, preprocess the image # called in ImageEncodingStage, preprocess the image
def preprocess_image(self, image, image_processor: VaeImageProcessor): def preprocess_image(self, image, image_processor: VaeImageProcessor):
return image return image
python/sglang/multimodal_gen/configs/pipelines/qwen_image.py
+1 -1
View File
@@ -273,7 +273,7 @@ class QwenImageEditPipelineConfig(QwenImagePipelineConfig):
image = image_processor.resize(image, calculated_height, calculated_width) image = image_processor.resize(image, calculated_height, calculated_width)
return image return image
def set_width_and_height(self, width, height, image): def adjust_size(self, width, height, image):
image_size = image[0].size if isinstance(image, list) else image.size image_size = image[0].size if isinstance(image, list) else image.size
calculated_width, calculated_height, _ = calculate_dimensions( calculated_width, calculated_height, _ = calculate_dimensions(
1024 * 1024, image_size[0] / image_size[1] 1024 * 1024, image_size[0] / image_size[1]
python/sglang/multimodal_gen/configs/pipelines/wan.py
+21 -2
View File
@@ -15,6 +15,9 @@ from sglang.multimodal_gen.configs.models.encoders import (
) )
from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig from sglang.multimodal_gen.configs.models.vaes import WanVAEConfig
from sglang.multimodal_gen.configs.pipelines.base import ModelTaskType, PipelineConfig from sglang.multimodal_gen.configs.pipelines.base import ModelTaskType, PipelineConfig
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tensor: def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tensor:
@@ -33,6 +36,22 @@ def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tenso
return prompt_embeds_tensor return prompt_embeds_tensor
@dataclass
class WanI2VCommonConfig(PipelineConfig):
# for all wan i2v pipelines
def adjust_num_frames(self, num_frames):
vae_scale_factor_temporal = self.vae_config.arch_config.scale_factor_temporal
if num_frames % vae_scale_factor_temporal != 1:
logger.warning(
f"`num_frames - 1` has to be divisible by {vae_scale_factor_temporal}. Rounding to the nearest number."
)
num_frames = (
num_frames // vae_scale_factor_temporal * vae_scale_factor_temporal + 1
)
return num_frames
return num_frames
@dataclass @dataclass
class WanT2V480PConfig(PipelineConfig): class WanT2V480PConfig(PipelineConfig):
"""Base configuration for Wan T2V 1.3B pipeline architecture.""" """Base configuration for Wan T2V 1.3B pipeline architecture."""
@@ -81,7 +100,7 @@ class WanT2V720PConfig(WanT2V480PConfig):
@dataclass @dataclass
class WanI2V480PConfig(WanT2V480PConfig): class WanI2V480PConfig(WanT2V480PConfig, WanI2VCommonConfig):
"""Base configuration for Wan I2V 14B 480P pipeline architecture.""" """Base configuration for Wan I2V 14B 480P pipeline architecture."""
# WanConfig-specific parameters with defaults # WanConfig-specific parameters with defaults
@@ -128,7 +147,7 @@ class FastWan2_1_T2V_480P_Config(WanT2V480PConfig):
@dataclass @dataclass
class Wan2_2_TI2V_5B_Config(WanT2V480PConfig): class Wan2_2_TI2V_5B_Config(WanT2V480PConfig, WanI2VCommonConfig):
flow_shift: float | None = 5.0 flow_shift: float | None = 5.0
task_type: ModelTaskType = ModelTaskType.TI2V task_type: ModelTaskType = ModelTaskType.TI2V
expand_timesteps: bool = True expand_timesteps: bool = True
python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py
+1 -1
View File
@@ -259,7 +259,7 @@ class DiffGenerator:
# TODO: simplify # TODO: simplify
data_type = ( data_type = (
DataType.IMAGE DataType.IMAGE
if self.server_args.pipeline_config.task_type.is_image_task() if self.server_args.pipeline_config.task_type.is_image_gen()
or pretrained_sampling_params.num_frames == 1 or pretrained_sampling_params.num_frames == 1
else DataType.VIDEO else DataType.VIDEO
) )
python/sglang/multimodal_gen/runtime/entrypoints/utils.py
+6 -2
View File
@@ -45,12 +45,12 @@ def prepare_sampling_params(
# Validate dimensions # Validate dimensions
if sampling_params.num_frames <= 0: if sampling_params.num_frames <= 0:
raise ValueError( raise ValueError(
f"Height, width, and num_frames must be positive integers, got " f"height, width, and num_frames must be positive integers, got "
f"height={sampling_params.height}, width={sampling_params.width}, " f"height={sampling_params.height}, width={sampling_params.width}, "
f"num_frames={sampling_params.num_frames}" f"num_frames={sampling_params.num_frames}"
) )
if pipeline_config.task_type.is_image_task(): if pipeline_config.task_type.is_image_gen():
# settle num_frames # settle num_frames
logger.debug(f"Setting num_frames to 1 because this is a image-gen model") logger.debug(f"Setting num_frames to 1 because this is a image-gen model")
sampling_params.num_frames = 1 sampling_params.num_frames = 1
@@ -104,6 +104,10 @@ def prepare_sampling_params(
) )
sampling_params.num_frames = new_num_frames sampling_params.num_frames = new_num_frames
sampling_params.num_frames = server_args.pipeline_config.adjust_num_frames(
sampling_params.num_frames
)
sampling_params.set_output_file_ext() sampling_params.set_output_file_ext()
sampling_params.log(server_args=server_args) sampling_params.log(server_args=server_args)
return sampling_params return sampling_params
python/sglang/multimodal_gen/runtime/pipelines/schedule_batch.py
+1 -1
View File
@@ -235,7 +235,7 @@ class Req:
def set_width_and_height(self, server_args: ServerArgs): def set_width_and_height(self, server_args: ServerArgs):
if self.height is None or self.width is None: if self.height is None or self.width is None:
width, height = server_args.pipeline_config.set_width_and_height( width, height = server_args.pipeline_config.adjust_size(
self.width, self.height, self.pil_image self.width, self.height, self.pil_image
) )
self.width = width self.width = width
python/sglang/multimodal_gen/runtime/pipelines/stages/input_validation.py
+1 -1
View File
@@ -123,7 +123,7 @@ class InputValidationStage(PipelineStage):
if isinstance(server_args.pipeline_config, QwenImageEditPipelineConfig): if isinstance(server_args.pipeline_config, QwenImageEditPipelineConfig):
height = None if batch.height_not_provided else batch.height height = None if batch.height_not_provided else batch.height
width = None if batch.width_not_provided else batch.width width = None if batch.width_not_provided else batch.width
width, height = server_args.pipeline_config.set_width_and_height( width, height = server_args.pipeline_config.adjust_size(
height, width, batch.pil_image height, width, batch.pil_image
) )
batch.width = width batch.width = width
python/sglang/multimodal_gen/runtime/server_args.py
+1 -1
View File
@@ -803,7 +803,7 @@ class ServerArgs:
def check_server_sp_args(self): def check_server_sp_args(self):
if self.pipeline_config.task_type.is_image_task(): if self.pipeline_config.task_type.is_image_gen():
if ( if (
(self.sp_degree and self.sp_degree > 1) (self.sp_degree and self.sp_degree > 1)
or (self.ulysses_degree and self.ulysses_degree > 1) or (self.ulysses_degree and self.ulysses_degree > 1)
python/sglang/multimodal_gen/test/server/diffusion_config.py
+16 -16
View File
@@ -180,22 +180,22 @@ DIFFUSION_CASES: list[DiffusionCase] = [
startup_grace_seconds=30.0, startup_grace_seconds=30.0,
custom_validator="video", custom_validator="video",
), ),
# # === Image to Video (I2V) === # === Image to Video (I2V) ===
# DiffusionCase( DiffusionCase(
# id="wan2_1_i2v_480p", id="wan2_2_i2v",
# model_path="Wan-AI/Wan2.1-I2V-14B-Diffusers", model_path="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
# scenario_name="image_to_video", scenario_name="image_to_video",
# modality="video", modality="video",
# prompt="generate", # passing in something since failing if no prompt is passed prompt="generate", # passing in something since failing if no prompt is passed
# warmup_text=0, # warmups only for image gen models warmup_text=0, # warmups only for image gen models
# warmup_edit=0, warmup_edit=0,
# output_size="1024x1536", output_size="832x1104",
# image_edit_prompt="generate", image_edit_prompt="generate",
# image_edit_path="https://github.com/lm-sys/lm-sys.github.io/releases/download/test/TI2I_Qwen_Image_Edit_Input.jpg", image_edit_path="https://github.com/Wan-Video/Wan2.2/blob/990af50de458c19590c245151197326e208d7191/examples/i2v_input.JPG?raw=true",
# startup_grace_seconds=30.0, startup_grace_seconds=30.0,
# custom_validator="video", custom_validator="video",
# seconds=4, seconds=1,
# ), ),
# === Text and Image to Video (TI2V) === # === Text and Image to Video (TI2V) ===
DiffusionCase( DiffusionCase(
id="wan2_2_ti2v_5b", id="wan2_2_ti2v_5b",
python/sglang/multimodal_gen/test/server/perf_baselines.json
+155 -142
View File
@@ -1,142 +1,155 @@
{ {
"metadata": { "metadata": {
"model": "Diffusion Server", "model": "Diffusion Server",
"hardware": "CI H100 80GB pool", "hardware": "CI H100 80GB pool",
"description": "Reference numbers captured from the CI diffusion server baseline run" "description": "Reference numbers captured from the CI diffusion server baseline run"
}, },
"tolerances": { "tolerances": {
"e2e": 0.25, "e2e": 0.25,
"stage": 0.3, "stage": 0.3,
"denoise_step": 0.1, "denoise_step": 0.2,
"denoise_agg": 0.1 "denoise_agg": 0.1
}, },
"sampling": { "sampling": {
"step_fractions": [ "step_fractions": [
0.0, 0.0,
0.2, 0.2,
0.4, 0.4,
0.6, 0.6,
0.8, 0.8,
1.0 1.0
], ],
"warmup_requests": { "warmup_requests": {
"text": 1, "text": 1,
"image_edit": 0 "image_edit": 0
} }
}, },
"scenarios": { "scenarios": {
"text_to_image": { "text_to_image": {
"notes": "Single-image generation using the default prompt", "notes": "Single-image generation using the default prompt",
"expected_e2e_ms": 74500.0, "expected_e2e_ms": 74500.0,
"expected_avg_denoise_ms": 422.42, "expected_avg_denoise_ms": 422.42,
"expected_median_denoise_ms": 410.62, "expected_median_denoise_ms": 410.62,
"stages_ms": { "stages_ms": {
"InputValidationStage": 0.1, "InputValidationStage": 0.1,
"TextEncodingStage": 834.2, "TextEncodingStage": 834.2,
"ConditioningStage": 0.1, "ConditioningStage": 0.1,
"TimestepPreparationStage": 10.6, "TimestepPreparationStage": 10.6,
"LatentPreparationStage": 9.0, "LatentPreparationStage": 9.0,
"DenoisingStage": 21202.6, "DenoisingStage": 21202.6,
"DecodingStage": 476.12 "DecodingStage": 476.12
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 1077.77, "1": 345.13, "2": 413.8, "3": 405.49, "4": 408.14, "5": 409.06, "0": 1077.77, "1": 345.13, "2": 413.8, "3": 405.49, "4": 408.14, "5": 409.06,
"6": 408.85, "7": 410.53, "8": 407.51, "9": 409.44, "10": 408.65, "11": 410.14, "6": 408.85, "7": 410.53, "8": 407.51, "9": 409.44, "10": 408.65, "11": 410.14,
"12": 411.74, "13": 409.59, "14": 409.17, "15": 410.78, "16": 410.66, "17": 410.58, "12": 411.74, "13": 409.59, "14": 409.17, "15": 410.78, "16": 410.66, "17": 410.58,
"18": 411.27, "19": 410.51, "20": 409.03, "21": 410.16, "22": 409.42, "23": 411.03, "18": 411.27, "19": 410.51, "20": 409.03, "21": 410.16, "22": 409.42, "23": 411.03,
"24": 410.18, "25": 409.72, "26": 410.26, "27": 410.21, "28": 410.71, "29": 410.76, "24": 410.18, "25": 409.72, "26": 410.26, "27": 410.21, "28": 410.71, "29": 410.76,
"30": 411.06, "31": 410.1, "32": 410.55, "33": 410.77, "34": 410.74, "35": 411.75, "30": 411.06, "31": 410.1, "32": 410.55, "33": 410.77, "34": 410.74, "35": 411.75,
"36": 410.78, "37": 411.56, "38": 410.85, "39": 411.08, "40": 411.12, "41": 411.1, "36": 410.78, "37": 411.56, "38": 410.85, "39": 411.08, "40": 411.12, "41": 411.1,
"42": 411.09, "43": 410.87, "44": 411.37, "45": 411.68, "46": 411.0, "47": 410.09, "42": 411.09, "43": 410.87, "44": 411.37, "45": 411.68, "46": 411.0, "47": 410.09,
"48": 412.72, "49": 410.42 "48": 412.72, "49": 410.42
} }
}, },
"image_edit": { "image_edit": {
"notes": "single uploaded reference image, Qwen/Qwen-Image-Edit", "notes": "single uploaded reference image, Qwen/Qwen-Image-Edit",
"expected_e2e_ms": 138500.0, "expected_e2e_ms": 138500.0,
"expected_avg_denoise_ms": 720.0, "expected_avg_denoise_ms": 720.0,
"expected_median_denoise_ms": 718.0, "expected_median_denoise_ms": 718.0,
"stages_ms": { "stages_ms": {
"InputValidationStage": 23, "InputValidationStage": 23,
"ImageEncodingStage": 990.0, "ImageEncodingStage": 1350.0,
"ImageVAEEncodingStage": 340.0, "ImageVAEEncodingStage": 340.0,
"ConditioningStage": 0.13, "ConditioningStage": 0.13,
"TimestepPreparationStage": 13.78, "TimestepPreparationStage": 13.78,
"LatentPreparationStage": 10.0, "LatentPreparationStage": 10.0,
"DenoisingStage": 36000.0, "DenoisingStage": 36000.0,
"DecodingStage": 645 "DecodingStage": 850.0
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 720.0, "1": 720.0, "2": 720.0, "3": 720.0, "4": 720.0, "5": 720.0, "0": 720.0, "1": 720.0, "2": 720.0, "3": 720.0, "4": 720.0, "5": 720.0,
"6": 720.0, "7": 720.0, "8": 720.0, "9": 720.0, "10": 720.0, "11": 720.0, "6": 720.0, "7": 720.0, "8": 720.0, "9": 720.0, "10": 720.0, "11": 720.0,
"12": 720.0, "13": 720.0, "14": 720.0, "15": 720.0, "16": 720.0, "17": 720.0, "12": 720.0, "13": 720.0, "14": 720.0, "15": 720.0, "16": 720.0, "17": 720.0,
"18": 720.0, "19": 720.0, "20": 720.0, "21": 720.0, "22": 720.0, "23": 720.0, "18": 720.0, "19": 720.0, "20": 720.0, "21": 720.0, "22": 720.0, "23": 720.0,
"24": 720.0, "25": 720.0, "26": 720.0, "27": 720.0, "28": 720.0, "29": 720.0, "24": 720.0, "25": 720.0, "26": 720.0, "27": 720.0, "28": 720.0, "29": 720.0,
"30": 720.0, "31": 720.0, "32": 720.0, "33": 720.0, "34": 720.0, "35": 720.0, "30": 720.0, "31": 720.0, "32": 720.0, "33": 720.0, "34": 720.0, "35": 720.0,
"36": 720.0, "37": 720.0, "38": 720.0, "39": 720.0, "40": 720.0, "41": 720.0, "36": 720.0, "37": 720.0, "38": 720.0, "39": 720.0, "40": 720.0, "41": 720.0,
"42": 720.0, "43": 720.0, "44": 720.0, "45": 720.0, "46": 720.0, "47": 720.0, "42": 720.0, "43": 720.0, "44": 720.0, "45": 720.0, "46": 720.0, "47": 720.0,
"48": 720.0, "49": 720.0 "48": 720.0, "49": 720.0
} }
}, },
"text_to_video": { "text_to_video": {
"notes": "Single-video generation using the default prompt", "notes": "Single-video generation using the default prompt",
"expected_e2e_ms": 95616.59, "expected_e2e_ms": 95616.59,
"expected_avg_denoise_ms": 1798.77, "expected_avg_denoise_ms": 1798.77,
"expected_median_denoise_ms": 1786.78, "expected_median_denoise_ms": 1786.78,
"stages_ms": { "stages_ms": {
"InputValidationStage": 1.03, "InputValidationStage": 1.03,
"TextEncodingStage": 3450.0, "TextEncodingStage": 3450.0,
"ConditioningStage": 1.0, "ConditioningStage": 1.0,
"TimestepPreparationStage": 6.0, "TimestepPreparationStage": 6.0,
"LatentPreparationStage": 15.0, "LatentPreparationStage": 15.0,
"DenoisingStage": 90100.0, "DenoisingStage": 90100.0,
"DecodingStage": 3650.0 "DecodingStage": 3650.0
}, },
"denoise_step_ms": { "denoise_step_ms": {
"0": 3500.0, "10": 1800.0, "20": 1800.0, "29": 1800.0, "39": 1800.0, "49": 1800.0 "0": 3500.0, "10": 1800.0, "20": 1800.0, "29": 1800.0, "39": 1800.0, "49": 1800.0
}, },
"frames_per_second": 0.51, "frames_per_second": 0.51,
"total_frames": 49, "total_frames": 49,
"avg_frame_time_ms": 1951.36 "avg_frame_time_ms": 1951.36
}, },
"image_to_video": { "image_to_video": {
"notes": "Image-to-Video generation baseline placeholder: TODO(bug)", "notes": "Wan-AI/Wan2.2-I2V-A14B",
"expected_e2e_ms": 1000000000.0, "expected_e2e_ms": 282500.0,
"expected_avg_denoise_ms": 1000000000.0, "expected_avg_denoise_ms": 7000.0,
"expected_median_denoise_ms": 1000000000.0, "expected_median_denoise_ms": 7000.19,
"stages_ms": {}, "stages_ms": {
"denoise_step_ms": {}, "InputValidationStage": 20.0,
"frames_per_second": null, "TextEncodingStage": 2100.0,
"total_frames": null, "ConditioningStage": 2.0,
"avg_frame_time_ms": null "TimestepPreparationStage": 2.0,
}, "LatentPreparationStage": 10.0,
"text_image_to_video": { "ImageVAEEncodingStage": 1800.0,
"notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B", "DenoisingStage": 278000.0,
"expected_e2e_ms": 178300.0, "DecodingStage": 2700.0
"expected_avg_denoise_ms": 3250.0, },
"expected_median_denoise_ms": 3260.0, "denoise_step_ms": {
"stages_ms": { "0": 24000.0,
"InputValidationStage": 80.0, "8": 7000.0,
"TextEncodingStage": 3000.0, "16": 7000.0,
"ConditioningStage": 1.0, "23": 7000.0,
"TimestepPreparationStage": 6.0, "31": 7000.0,
"LatentPreparationStage": 30.0, "39": 7000.0
"DenoisingStage": 162900.0, }
"DecodingStage": 13500.0 },
}, "text_image_to_video": {
"denoise_step_ms": { "notes": "Text-and-Image-to-Video generation baseline for Wan2.2-TI2V-5B",
"0": 3700.0, "expected_e2e_ms": 178300.0,
"10": 3300.0, "expected_avg_denoise_ms": 3250.0,
"20": 3300.0, "expected_median_denoise_ms": 3260.0,
"29": 3300.0, "stages_ms": {
"39": 3300.0, "InputValidationStage": 80.0,
"49": 3300.0 "TextEncodingStage": 3000.0,
}, "ConditioningStage": 1.0,
"frames_per_second": null, "TimestepPreparationStage": 6.0,
"total_frames": null, "LatentPreparationStage": 30.0,
"avg_frame_time_ms": null "DenoisingStage": 162900.0,
} "DecodingStage": 13500.0
} },
} "denoise_step_ms": {
"0": 3700.0,
"10": 3300.0,
"20": 3300.0,
"29": 3300.0,
"39": 3300.0,
"49": 3300.0
},
"frames_per_second": null,
"total_frames": null,
"avg_frame_time_ms": null
}
}
}