diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/diffusion_scheduler_utils.py b/python/sglang/multimodal_gen/runtime/pipelines_core/diffusion_scheduler_utils.py index d61be35a4c..6322906f76 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/diffusion_scheduler_utils.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/diffusion_scheduler_utils.py @@ -2,8 +2,9 @@ from __future__ import annotations +import inspect from copy import deepcopy -from typing import Any +from typing import Any, List, Optional, Union import torch @@ -24,6 +25,50 @@ def calculate_linear_shift( return image_seq_len * slope + base_shift - slope * base_seq_len +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """Set a step count or one custom schedule and return timesteps and step count.""" + if timesteps is not None and sigmas is not None: + raise ValueError( + "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values" + ) + if timesteps is not None: + accepts_timesteps = "timesteps" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys() + ) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys() + ) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + def clone_scheduler_runtime(scheduler: Any) -> Any: """Create an isolated scheduler runtime from a scheduler template or runtime.""" return deepcopy(scheduler) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py index e2c7c8fb02..72cf0868c2 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/hunyuan3d/shape.py @@ -24,6 +24,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.component_loading import ( load_transformer_if_needed, register_loaded_transformer, ) +from sglang.multimodal_gen.runtime.pipelines_core.diffusion_scheduler_utils import ( + retrieve_timesteps, +) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( @@ -43,51 +46,6 @@ from sglang.multimodal_gen.runtime.utils.mesh3d_utils import export_to_trimesh logger = init_logger(__name__) -def retrieve_timesteps( - scheduler, - num_inference_steps=None, - device=None, - timesteps=None, - sigmas=None, - **kwargs, -): - """Retrieve timesteps from scheduler.""" - import inspect - - if timesteps is not None and sigmas is not None: - raise ValueError("Only one of timesteps or sigmas can be passed.") - - if timesteps is not None: - accepts_timesteps = "timesteps" in set( - inspect.signature(scheduler.set_timesteps).parameters.keys() - ) - if not accepts_timesteps: - raise ValueError( - f"Scheduler {scheduler.__class__} doesn't support custom timesteps." - ) - scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) - timesteps = scheduler.timesteps - num_inference_steps = len(timesteps) - - elif sigmas is not None: - accepts_sigmas = "sigmas" in set( - inspect.signature(scheduler.set_timesteps).parameters.keys() - ) - if not accepts_sigmas: - raise ValueError( - f"Scheduler {scheduler.__class__} doesn't support custom sigmas." - ) - scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) - timesteps = scheduler.timesteps - num_inference_steps = len(timesteps) - - else: - scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) - timesteps = scheduler.timesteps - - return timesteps, num_inference_steps - - def _prepare_shape_image(image_processor, image, mask=None) -> dict: """Prepare shape image for conditioning.""" if isinstance(image, torch.Tensor) and isinstance(mask, torch.Tensor): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py index 19f47a9ac7..2f98078f7c 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/qwen_image_layered.py @@ -1,4 +1,3 @@ -import inspect import math from typing import List, Optional, Union @@ -12,6 +11,9 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) +from sglang.multimodal_gen.runtime.pipelines_core.diffusion_scheduler_utils import ( + retrieve_timesteps, +) from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage from sglang.multimodal_gen.runtime.server_args import ServerArgs @@ -89,72 +91,6 @@ def retrieve_latents( return encoder_output -# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps -def retrieve_timesteps( - scheduler, - num_inference_steps: Optional[int] = None, - device: Optional[Union[str, torch.device]] = None, - timesteps: Optional[List[int]] = None, - sigmas: Optional[List[float]] = None, - **kwargs, -): - r""" - Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles - custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. - - Args: - scheduler (`SchedulerMixin`): - The scheduler to get timesteps from. - num_inference_steps (`int`): - The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` - must be `None`. - device (`str` or `torch.device`, *optional*): - The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. - timesteps (`List[int]`, *optional*): - Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, - `num_inference_steps` and `sigmas` must be `None`. - sigmas (`List[float]`, *optional*): - Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, - `num_inference_steps` and `timesteps` must be `None`. - - Returns: - `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the - second element is the number of inference steps. - """ - if timesteps is not None and sigmas is not None: - raise ValueError( - "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values" - ) - if timesteps is not None: - accepts_timesteps = "timesteps" in set( - inspect.signature(scheduler.set_timesteps).parameters.keys() - ) - if not accepts_timesteps: - raise ValueError( - f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" timestep schedules. Please check whether you are using the correct scheduler." - ) - scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) - timesteps = scheduler.timesteps - num_inference_steps = len(timesteps) - elif sigmas is not None: - accept_sigmas = "sigmas" in set( - inspect.signature(scheduler.set_timesteps).parameters.keys() - ) - if not accept_sigmas: - raise ValueError( - f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" - f" sigmas schedules. Please check whether you are using the correct scheduler." - ) - scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) - timesteps = scheduler.timesteps - num_inference_steps = len(timesteps) - else: - scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) - timesteps = scheduler.timesteps - return timesteps, num_inference_steps - - class QwenImageLayeredBeforeDenoisingStage(PipelineStage): def __init__( self, diff --git a/python/sglang/multimodal_gen/test/unit/test_pi05_runtime_helpers.py b/python/sglang/multimodal_gen/test/unit/test_pi05_runtime_helpers.py index 733885e5e1..6deecaf573 100644 --- a/python/sglang/multimodal_gen/test/unit/test_pi05_runtime_helpers.py +++ b/python/sglang/multimodal_gen/test/unit/test_pi05_runtime_helpers.py @@ -560,8 +560,8 @@ def test_pi05_siglip_reuses_srt_model_with_layerwise_groups(): assert vision_model.embeddings.position_embedding.tp_size == 1 assert layer.self_attn.tp_size == 1 assert layer.self_attn.qkv_backend.flatten_batch is False - assert layer.mlp.fc1.tp_size == 1 - assert layer.mlp.fc2.tp_size == 1 + assert layer.mlp.fc1.output_size_per_partition == config.intermediate_size + assert layer.mlp.fc2.input_size_per_partition == config.intermediate_size assert isinstance(layer.mlp.act, nn.GELU) assert layer.mlp.act.approximate == "tanh" assert model.device == vision_model.embeddings.patch_embedding.weight.device