From e6931ca88991c4b1bd9d633b5e4232e5611e7cc8 Mon Sep 17 00:00:00 2001 From: ronnie_zheng Date: Mon, 21 Sep 2026 20:45:53 +0300 Subject: [PATCH] [Diffusion] migrate the whole _register_configs from registry.py to the model own config file (#40475) --- .../sglang-diffusion/support_new_models.mdx | 45 +- .../configs/pipeline_configs/cosmos3.py | 21 + .../configs/pipeline_configs/ernie_image.py | 19 + .../configs/pipeline_configs/flux.py | 58 ++ .../configs/pipeline_configs/glm_image.py | 13 + .../configs/pipeline_configs/helios.py | 38 + .../configs/pipeline_configs/hunyuan.py | 24 + .../configs/pipeline_configs/hunyuan3d.py | 16 + .../configs/pipeline_configs/ideogram.py | 36 + .../configs/pipeline_configs/joy_echo.py | 21 + .../configs/pipeline_configs/joy_image.py | 18 + .../configs/pipeline_configs/krea2.py | 14 + .../pipeline_configs/lingbot_video_moe.py | 15 + .../configs/pipeline_configs/lingbot_world.py | 23 + .../configs/pipeline_configs/longcat_image.py | 49 + .../configs/pipeline_configs/longlive2.py | 16 + .../configs/pipeline_configs/ltx_2.py | 34 + .../configs/pipeline_configs/ltx_2_5.py | 14 + .../configs/pipeline_configs/minimax_h3.py | 35 + .../pipeline_configs/minimax_h3_vdn.py | 21 + .../configs/pipeline_configs/mova.py | 23 + .../configs/pipeline_configs/pi05.py | 18 + .../configs/pipeline_configs/qwen_image.py | 69 ++ .../configs/pipeline_configs/qwen_image21.py | 14 + .../configs/pipeline_configs/sana.py | 27 + .../configs/pipeline_configs/sana_video.py | 16 + .../configs/pipeline_configs/sana_wm.py | 17 + .../configs/pipeline_configs/sensenova_u1.py | 15 + .../pipeline_configs/stablediffusion3.py | 30 + .../configs/pipeline_configs/wan.py | 111 +++ .../configs/pipeline_configs/zimage.py | 27 + python/sglang/multimodal_gen/registry.py | 896 ++---------------- 32 files changed, 936 insertions(+), 857 deletions(-) diff --git a/docs/docs/sglang-diffusion/support_new_models.mdx b/docs/docs/sglang-diffusion/support_new_models.mdx index c40f3401b..5733f9cd7 100644 --- a/docs/docs/sglang-diffusion/support_new_models.mdx +++ b/docs/docs/sglang-diffusion/support_new_models.mdx @@ -12,12 +12,11 @@ though the runtime is split into separate folders. The files are split by runtime responsibility. For a new model, read the request path first: -1. `registry.py` chooses the model family, sampling params, and pipeline config. -2. `configs/pipeline_configs/{model}.py` defines model-specific denoising and +1. `configs/pipeline_configs/{model}.py` defines model-specific denoising and decoding behavior. -3. `runtime/pipelines/{model}.py` wires modules into stages. -4. `runtime/pipelines_core/stages/` runs the shared stage logic. -5. `runtime/models/` contains native model components only when the architecture +2. `runtime/pipelines/{model}.py` wires modules into stages. +3. `runtime/pipelines_core/stages/` runs the shared stage logic. +4. `runtime/models/` contains native model components only when the architecture cannot be reused. That is the dependency direction. Avoid making a model PR that requires readers @@ -124,7 +123,6 @@ behavior. | Area | Add or edit when | Typical file | | --- | --- | --- | -| Registry | Always, unless extending an already registered family | `python/sglang/multimodal_gen/registry.py` | | Runtime parameters | The request schema differs from existing models | `configs/sample/{model}.py` | | Pipeline config | Denoising, decoding, precision, position encoding, or CFG hooks differ | `configs/pipeline_configs/{model}.py` | | Pipeline wiring | The model needs a new stage layout or module list | `runtime/pipelines/{model}.py` | @@ -137,11 +135,10 @@ behavior. For a new native architecture, the common minimum is: -1. `registry.py` -2. `configs/sample/{model}.py` -3. `configs/pipeline_configs/{model}.py` -4. `runtime/pipelines/{model}.py` -5. `runtime/models/dits/{model}.py` +1. `configs/sample/{model}.py` +2. `configs/pipeline_configs/{model}.py` +3. `runtime/pipelines/{model}.py` +4. `runtime/models/dits/{model}.py` Every extra file should map to model behavior that existing code cannot express clearly. @@ -499,19 +496,27 @@ native integration contract. ### 6. Registry -Register the family once the sampling params and pipeline config exist. +Define a `register()` function in `configs/pipeline_configs/{model}.py`. The +runtime auto-discovers it on startup and calls it to register the sampling +params and pipeline config. ```python -register_configs( - model_family="my_model", - sampling_param_cls=MyModelSamplingParams, - pipeline_config_cls=MyModelPipelineConfig, - hf_model_paths=["org/my-model"], -) +# python/sglang/multimodal_gen/configs/pipeline_configs/my_model.py +def register(): + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=MyModelSamplingParams, + pipeline_config_cls=MyModelPipelineConfig, + hf_model_paths=["org/my-model"], + model_detectors=[lambda hf_id: "my-model" in hf_id.lower()], + ) ``` -The pipeline file is discovered through its `EntryClass`; do not add a second -pipeline registry unless the existing registry requires it. +`model_detectors` matches a model path or `model_index.json` `_class_name` when +the Hugging Face path varies; see `wan.py` or `qwen_image21.py` for real +examples. The pipeline file is discovered through its `EntryClass`; do not add +a second pipeline registry unless the existing registry requires it. ## Verify the Port diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py index f0321b46d..fc692380a 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/cosmos3.py @@ -202,3 +202,24 @@ class Cosmos3Config(PipelineConfig): keep_resident_min_available_gb=threshold_gb, keep_resident_components=("dit", "vae"), ) + + +def register(): + from sglang.multimodal_gen.configs.sample.cosmos3 import ( + Cosmos3SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Cosmos3SamplingParams, + pipeline_config_cls=Cosmos3Config, + hf_model_paths=[ + "nvidia/Cosmos3-Nano", + "nvidia/Cosmos3-Nano-Policy-DROID", + "nvidia/Cosmos3-Super", + "nvidia/Cosmos3-Super-Text2Image", + "nvidia/Cosmos3-Super-Image2Video", + "nvidia/Cosmos3-Edge", + ], + model_detectors=[lambda hf_id: "cosmos3omni" in hf_id.lower()], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ernie_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ernie_image.py index 4f4315f72..c7e892717 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/ernie_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ernie_image.py @@ -258,3 +258,22 @@ class ErnieImagePipelineConfig(ImagePipelineConfig): def post_denoising_loop(self, latents, batch): return latents + + +def register(): + from sglang.multimodal_gen.configs.sample.ernie_image import ( + ErnieImageSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=ErnieImageSamplingParams, + pipeline_config_cls=ErnieImagePipelineConfig, + hf_model_paths=[ + "baidu/ERNIE-Image", + "baidu/ERNIE-Image-Turbo", + ], + model_detectors=[ + lambda hf_id: "ernie-image" in hf_id.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py index bbb2d4023..0b344904b 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/flux.py @@ -823,3 +823,61 @@ class Flux2KleinBasePipelineConfig(Flux2KleinPipelineConfig): txt_seq_lens, ) } + + +def register(): + from sglang.multimodal_gen.configs.sample.flux import ( + Flux2KleinBaseSamplingParams, + Flux2KleinSamplingParams, + Flux2SamplingParams, + FluxSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=FluxSamplingParams, + pipeline_config_cls=FluxPipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.1-dev", + ], + model_detectors=[lambda hf_id: "flux.1" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=Flux2KleinSamplingParams, + pipeline_config_cls=Flux2KleinPipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.2-klein-4B", + "black-forest-labs/FLUX.2-klein-9B", + ], + model_detectors=[ + lambda hf_id: ( + ("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower()) + and "base" not in hf_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=Flux2KleinBaseSamplingParams, + pipeline_config_cls=Flux2KleinBasePipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.2-klein-base-4B", + "black-forest-labs/FLUX.2-klein-base-9B", + ], + model_detectors=[ + lambda hf_id: ( + ("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower()) + and "base" in hf_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=Flux2SamplingParams, + pipeline_config_cls=Flux2PipelineConfig, + hf_model_paths=[ + "black-forest-labs/FLUX.2-dev", + "black-forest-labs/FLUX.2-dev-NVFP4", + ], + model_detectors=[ + lambda hf_id: "flux.2" in hf_id.lower() and "klein" not in hf_id.lower() + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/glm_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/glm_image.py index a58fd8fec..5a3aca023 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/glm_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/glm_image.py @@ -118,3 +118,16 @@ class GlmImagePipelineConfig(SpatialImagePipelineConfig): def post_decoding(self, frames, server_args): return self.image_processor.postprocess(frames, output_type="latent") + + +def register(): + from sglang.multimodal_gen.configs.sample.glmimage import ( + GlmImageSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=GlmImageSamplingParams, + pipeline_config_cls=GlmImagePipelineConfig, + model_detectors=[lambda hf_id: "glm-image" in hf_id.lower()], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/helios.py b/python/sglang/multimodal_gen/configs/pipeline_configs/helios.py index 4e02c43c7..47e0b94d1 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/helios.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/helios.py @@ -128,3 +128,41 @@ class HeliosDistilledConfig(HeliosT2VConfig): pyramid_num_inference_steps_list: list[int] = field( default_factory=lambda: [10, 10, 10] ) + + +def register(): + from sglang.multimodal_gen.configs.sample.helios import ( + HeliosDistilledSamplingParams, + HeliosMidSamplingParams, + HeliosT2VSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=HeliosT2VSamplingParams, + pipeline_config_cls=HeliosT2VConfig, + hf_model_paths=[ + "BestWishYsh/Helios-Base", + ], + model_detectors=[ + lambda hf_id: ( + "helios" in hf_id.lower() + and "mid" not in hf_id.lower() + and "distill" not in hf_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=HeliosMidSamplingParams, + pipeline_config_cls=HeliosMidConfig, + hf_model_paths=[ + "BestWishYsh/Helios-Mid", + ], + ) + register_configs( + sampling_param_cls=HeliosDistilledSamplingParams, + pipeline_config_cls=HeliosDistilledConfig, + hf_model_paths=[ + "BestWishYsh/Helios-Distilled", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py index 0287b44ed..054b0745c 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan.py @@ -169,3 +169,27 @@ class FastHunyuanConfig(HunyuanConfig): keep_resident_min_available_gb=60, keep_resident_components=("dit", "vae"), ) + + +def register(): + from sglang.multimodal_gen.configs.sample.hunyuan import ( + FastHunyuanSamplingParam, + HunyuanSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=HunyuanSamplingParams, + pipeline_config_cls=HunyuanConfig, + hf_model_paths=[ + "hunyuanvideo-community/HunyuanVideo", + ], + model_detectors=[lambda hf_id: "hunyuanvideo" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=FastHunyuanSamplingParam, + pipeline_config_cls=FastHunyuanConfig, + hf_model_paths=[ + "FastVideo/FastHunyuan-diffusers", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py index 41903630f..028ff65e5 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/hunyuan3d.py @@ -78,3 +78,19 @@ class Hunyuan3D2PipelineConfig(PipelineConfig): latent_shape = self.vae_config.arch_config.latent_shape shape = (batch_size, *latent_shape) return shape + + +def register(): + from sglang.multimodal_gen.configs.sample.hunyuan3d import ( + Hunyuan3DSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Hunyuan3DSamplingParams, + pipeline_config_cls=Hunyuan3D2PipelineConfig, + hf_model_paths=[ + "tencent/Hunyuan3D-2", + ], + model_detectors=[lambda hf_id: "hunyuan3d" in hf_id.lower()], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ideogram.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ideogram.py index 946c2b87c..2e3f5bf7c 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/ideogram.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ideogram.py @@ -308,3 +308,39 @@ class Ideogram4PipelineConfig(ImagePipelineConfig): @dataclass class Ideogram4DistilledPipelineConfig(Ideogram4PipelineConfig): dit_config: DiTConfig = field(default_factory=Ideogram4DistilledDiTConfig) + + +def register(): + from sglang.multimodal_gen.configs.sample.ideogram import ( + Ideogram4FastSamplingParams, + Ideogram4InstantSamplingParams, + Ideogram4SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Ideogram4FastSamplingParams, + pipeline_config_cls=Ideogram4DistilledPipelineConfig, + hf_model_paths=["fal/ideogram-v4-fast"], + ) + register_configs( + sampling_param_cls=Ideogram4InstantSamplingParams, + pipeline_config_cls=Ideogram4DistilledPipelineConfig, + hf_model_paths=["fal/ideogram-v4-instant"], + ) + register_configs( + sampling_param_cls=Ideogram4SamplingParams, + pipeline_config_cls=Ideogram4PipelineConfig, + hf_model_paths=[ + "ideogram-ai/ideogram-4-fp8", + "ideogram-ai/ideogram-4-nf4", + "Comfy-Org/Ideogram-4", + ], + model_detectors=[ + lambda hf_id: "ideogram4pipeline" in hf_id.lower(), + lambda hf_id: "ideogram-4-fp8" in hf_id.lower(), + lambda hf_id: "ideogram-4-nf4" in hf_id.lower(), + lambda hf_id: "comfy-org/ideogram-4" in hf_id.lower(), + lambda hf_id: "comfy-org--ideogram-4" in hf_id.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_echo.py b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_echo.py index 913172df7..b761cb1b2 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_echo.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_echo.py @@ -58,3 +58,24 @@ class JoyEchoPipelineConfig(LTX2PipelineConfig): video_memory_frame_selection_mode: str = "center" late_layer_ratio: float = 0.7 + + +def register(): + from sglang.multimodal_gen.configs.sample.joy_echo import ( + JoyEchoSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=JoyEchoSamplingParams, + pipeline_config_cls=JoyEchoPipelineConfig, + hf_model_paths=[ + "jdopensource/JoyAI-Echo", + ], + model_detectors=[ + lambda hf_id: ( + ("joy-echo" in hf_id.lower() or "joyai-echo" in hf_id.lower()) + and "image-edit" not in hf_id.lower() + ), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py index 0760dc127..2e68c4851 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/joy_image.py @@ -428,3 +428,21 @@ class JoyImageEditPipelineConfig(ImagePipelineConfig): cond_norm = torch.norm(noise_pred_cond, dim=2, keepdim=True) noise_norm = torch.norm(noise_pred, dim=2, keepdim=True).clamp_min(1e-12) return noise_pred * (cond_norm / noise_norm) + + +def register(): + from sglang.multimodal_gen.configs.sample.joy_image import ( + JoyImageEditSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=JoyImageEditSamplingParams, + pipeline_config_cls=JoyImageEditPipelineConfig, + hf_model_paths=[ + "jdopensource/JoyAI-Image-Edit-Diffusers", + ], + model_detectors=[ + lambda hf_id: "joyai-image-edit" in hf_id.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py index e08af33c1..104ce8245 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/krea2.py @@ -163,3 +163,17 @@ class Krea2PipelineConfig(ImagePipelineConfig): ) latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) return latents + + +def register(): + from sglang.multimodal_gen.configs.sample.krea2 import ( + Krea2SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Krea2SamplingParams, + pipeline_config_cls=Krea2PipelineConfig, + hf_model_paths=["krea/Krea-2"], + model_detectors=[lambda hf_id: "krea-2" in hf_id.lower()], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_video_moe.py b/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_video_moe.py index 4f7bfd400..acf78067f 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_video_moe.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_video_moe.py @@ -84,3 +84,18 @@ class LingBotVideoMoEPipelineConfig(PipelineConfig): 1, -1, 1, 1, 1 ) return 1.0 / std, mean + + +def register(): + from sglang.multimodal_gen.configs.sample.lingbot_video_moe import ( + LingBotVideoMoESamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=LingBotVideoMoESamplingParams, + pipeline_config_cls=LingBotVideoMoEPipelineConfig, + model_detectors=[ + lambda hf_id: "lingbot-video-moe" in hf_id.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_world.py b/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_world.py index bb9889df2..81f0808b2 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_world.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/lingbot_world.py @@ -449,3 +449,26 @@ class LingBotWorldV2CausalDMDConfig(LingBotWorldCausalDMDConfig): dmd_denoising_steps: list[int] | None = field( default_factory=lambda: [1000, 750, 500, 250] ) + + +def register(): + from sglang.multimodal_gen.configs.sample.lingbot_world import ( + LingBotWorldSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=LingBotWorldSamplingParams, + pipeline_config_cls=LingBotWorldCausalDMDConfig, + hf_model_paths=[ + "IPostYellow/lingbot-world-fast-diffusers", + "robbyant/lingbot-world-fast-diffusers", + ], + ) + register_configs( + sampling_param_cls=LingBotWorldSamplingParams, + pipeline_config_cls=LingBotWorldV2CausalDMDConfig, + hf_model_paths=[ + "robbyant/lingbot-world-v2-14b-causal-fast-diffusers", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py index 098d0f29e..822e2a637 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/longcat_image.py @@ -604,3 +604,52 @@ class LongCatImageEditPipelineConfig(LongCatImagePipelineConfig): if latents.shape[1] > expected: latents = latents[:, :expected, :] return super().post_denoising_loop(latents, batch) + + +def register(): + from sglang.multimodal_gen.configs.sample.longcat_image import ( + LongCatImageEditSamplingParams, + LongCatImageEditTurboSamplingParams, + LongCatImageSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=LongCatImageSamplingParams, + pipeline_config_cls=LongCatImagePipelineConfig, + hf_model_paths=[ + "meituan-longcat/LongCat-Image", + ], + model_detectors=[ + lambda hf_id: "longcat" in hf_id.lower() and "edit" not in hf_id.lower(), + ], + ) + # LongCat-Image-Edit-Turbo (registered before Edit so its detector wins) + register_configs( + sampling_param_cls=LongCatImageEditTurboSamplingParams, + pipeline_config_cls=LongCatImageEditPipelineConfig, + hf_model_paths=[ + "meituan-longcat/LongCat-Image-Edit-Turbo", + ], + model_detectors=[ + lambda hf_id: ( + "longcat" in hf_id.lower() + and "edit" in hf_id.lower() + and "turbo" in hf_id.lower() + ), + ], + ) + register_configs( + sampling_param_cls=LongCatImageEditSamplingParams, + pipeline_config_cls=LongCatImageEditPipelineConfig, + hf_model_paths=[ + "meituan-longcat/LongCat-Image-Edit", + ], + model_detectors=[ + lambda hf_id: ( + "longcat" in hf_id.lower() + and "edit" in hf_id.lower() + and "turbo" not in hf_id.lower() + ), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py index 76318eae4..bb74ee0be 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py @@ -70,3 +70,19 @@ class LongLive2T2VConfig(Wan2_2_TI2V_5B_Config): super().__post_init__() self.vae_config.load_encoder = True self.vae_config.load_decoder = True + + +def register(): + from sglang.multimodal_gen.configs.sample.longlive2 import ( + LongLive2SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=LongLive2SamplingParams, + pipeline_config_cls=LongLive2T2VConfig, + hf_model_paths=[ + "Rabinovich/LongLive-2.0-5B-Diffusers", + "Efficient-Large-Model/LongLive-2.0-5B", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py index 240c23023..426adf593 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2.py @@ -726,3 +726,37 @@ class LTX23PipelineConfig(LTX2PipelineConfig): # original-mode lora swaps invalidate post-warmup timing calibration supports_auto_residency: bool = False + + +def register(): + from sglang.multimodal_gen.configs.sample.ltx_2 import ( + LTX2SamplingParams, + LTX23HQSamplingParams, + LTX23SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=LTX2SamplingParams, + pipeline_config_cls=LTX2PipelineConfig, + hf_model_paths=["Lightricks/LTX-2"], + model_detectors=[ + lambda path: "ltx" in path.lower() and "video" in path.lower(), + lambda path: ( + "ltx-2" in path.lower() + and "ltx-2.3" not in path.lower() + and "ltx-2.5" not in path.lower() + ), + ], + ) + register_configs( + sampling_param_cls=LTX23SamplingParams, + pipeline_config_cls=LTX23PipelineConfig, + hf_model_paths=["Lightricks/LTX-2.3"], + model_detectors=[ + lambda path: "ltx-2.3" in path.lower(), + ], + pipeline_config_registry_entries={ + "LTX2TwoStageHQPipeline": (LTX2PipelineConfig, LTX23HQSamplingParams), + }, + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2_5.py b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2_5.py index 10e493329..0732e9d6c 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2_5.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/ltx_2_5.py @@ -55,3 +55,17 @@ class LTX25PipelineConfig(LTX2PipelineConfig): default_sigmas: tuple[float, ...] | None = field( default_factory=lambda: LTX25_DISTILLED_SIGMA_VALUES ) + + +def register(): + from sglang.multimodal_gen.configs.sample.ltx_2_5 import LTX25SamplingParams + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=LTX25SamplingParams, + pipeline_config_cls=LTX25PipelineConfig, + hf_model_paths=["Lightricks/LTX-2.5-Diffusers"], + model_detectors=[ + lambda path: "ltx-2.5" in path.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py index 9d9b47757..7e30540a9 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3.py @@ -356,3 +356,38 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig): __all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"] + + +def register(): + from sglang.multimodal_gen.configs.sample.minimax_h3 import ( + FastH3SamplingParams, + MiniMaxH3SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=MiniMaxH3SamplingParams, + pipeline_config_cls=MiniMaxH3PipelineConfig, + hf_model_paths=[ + "MiniMaxAI/MiniMax-H3", + "MiniMax/MiniMax-H3", + ], + model_detectors=[ + lambda model_id: ( + "minimaxh3" in model_id.lower().replace("-", "").replace("_", "") + and "vdn" not in model_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=FastH3SamplingParams, + pipeline_config_cls=FastH3PipelineConfig, + hf_model_paths=[ + "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree", + ], + model_detectors=[ + lambda model_id: ( + "fasth3" in model_id.lower().replace("-", "").replace("_", "") + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py index 4267a92bd..9a9e0f0c6 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/minimax_h3_vdn.py @@ -74,3 +74,24 @@ class VDNH3PipelineConfig(MiniMaxH3PipelineConfig): __all__ = ["VDNH3PipelineConfig"] + + +def register(): + from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import ( + VDNH3SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=VDNH3SamplingParams, + pipeline_config_cls=VDNH3PipelineConfig, + hf_model_paths=[ + "OpenVDN/vdn-minimax-h3", + ], + model_detectors=[ + lambda model_id: ( + "vdn" in model_id.lower() + and "minimaxh3" in model_id.lower().replace("-", "").replace("_", "") + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py b/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py index c83936c3a..7e09301bc 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/mova.py @@ -195,3 +195,26 @@ class MOVA720PConfig(MOVAPipelineConfig): """Configuration for MOVA 720P (text+image -> video+audio) pipelines.""" max_area: int = 720 * 1280 + + +def register(): + from sglang.multimodal_gen.configs.sample.mova import ( + MOVA_360P_SamplingParams, + MOVA_720P_SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=MOVA_360P_SamplingParams, + pipeline_config_cls=MOVA360PConfig, + model_detectors=[ + lambda hf_id: "mova" in hf_id.lower() and "360p" in hf_id.lower() + ], + ) + register_configs( + sampling_param_cls=MOVA_720P_SamplingParams, + pipeline_config_cls=MOVA720PConfig, + model_detectors=[ + lambda hf_id: "mova" in hf_id.lower() and "720p" in hf_id.lower() + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py b/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py index 4d19069f4..3a7ce7b73 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/pi05.py @@ -164,3 +164,21 @@ class Pi05PipelineConfig(PipelineConfig): def get_model_deployment_config(self) -> ModelDeploymentConfig: return ModelDeploymentConfig() + + +def register(): + from sglang.multimodal_gen.configs.sample.pi05 import Pi05SamplingParams + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=Pi05SamplingParams, + pipeline_config_cls=Pi05PipelineConfig, + hf_model_paths=[ + "lerobot/pi05_base", + "lerobot/pi05_libero_base", + ], + model_detectors=[ + lambda hf_id: "pi05" in hf_id.lower(), + lambda hf_id: "pi0.5" in hf_id.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py index 7b39cfcdc..c99622e35 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image.py @@ -895,3 +895,72 @@ class QwenImageLayeredPipelineConfig(QwenImageEditPipelineConfig): latents = latents.permute(0, 2, 1, 3, 4).view(-1, c, 1, h, w) # latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) return latents + + +def register(): + from sglang.multimodal_gen.configs.sample.qwenimage import ( + QwenImage2512SamplingParams, + QwenImageEditPlusSamplingParams, + QwenImageLayeredSamplingParams, + QwenImageSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=QwenImageSamplingParams, + pipeline_config_cls=QwenImagePipelineConfig, + hf_model_paths=["Qwen/Qwen-Image", "nvidia/Qwen-Image-NVFP4"], + model_detectors=[ + lambda hf_id: ( + "qwen-image" in hf_id.lower() + and "edit" not in hf_id.lower() + and "layered" not in hf_id.lower() + and "2512" not in hf_id.lower() + and "qwen-image-2.1" not in hf_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=QwenImage2512SamplingParams, + pipeline_config_cls=QwenImagePipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-2512"], + model_detectors=[lambda hf_id: "qwen-image-2512" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=QwenImageSamplingParams, + pipeline_config_cls=QwenImageEditPipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-Edit"], + model_detectors=[ + lambda hf_id: ( + "qwen-image-edit" in hf_id.lower() + and "2509" not in hf_id.lower() + and "2511" not in hf_id.lower() + ) + ], + ) + register_configs( + sampling_param_cls=QwenImageEditPlusSamplingParams, + pipeline_config_cls=QwenImageEditPlusPipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-Edit-2509"], + model_detectors=[lambda hf_id: "qwen-image-edit-2509" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=QwenImageEditPlusSamplingParams, + pipeline_config_cls=QwenImageEditPlus_2511_PipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-Edit-2511"], + model_detectors=[lambda hf_id: "qwen-image-edit-2511" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=QwenImageLayeredSamplingParams, + pipeline_config_cls=QwenImageLayeredPipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-Layered"], + model_detectors=[lambda hf_id: "qwen-image-layered" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=QwenImageEditPlusSamplingParams, + pipeline_config_cls=QwenImageEditPlusPipelineConfig, + hf_model_paths=[ + "FireRedTeam/FireRed-Image-Edit-1.0", + "FireRedTeam/FireRed-Image-Edit-1.1", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image21.py b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image21.py index b06ede3b4..f69371f6d 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image21.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/qwen_image21.py @@ -86,3 +86,17 @@ class QwenImage21PipelineConfig(ImagePipelineConfig): def preprocess_condition_image(self, image, **kwargs): return image + + +def register(): + from sglang.multimodal_gen.configs.sample.qwenimage21 import ( + QwenImage21SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=QwenImage21SamplingParams, + pipeline_config_cls=QwenImage21PipelineConfig, + hf_model_paths=["Qwen/Qwen-Image-2.1"], + model_detectors=[lambda hf_id: "qwen-image-2.1" in hf_id.lower()], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sana.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sana.py index c3967b40e..66caa254a 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/sana.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sana.py @@ -127,3 +127,30 @@ class SanaPipelineConfig(SpatialImagePipelineConfig): def gather_latents_for_sp(self, latents): return latents + + +def register(): + from sglang.multimodal_gen.configs.sample.sana import SanaSamplingParams + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=SanaSamplingParams, + pipeline_config_cls=SanaPipelineConfig, + hf_model_paths=[ + "Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers", + "Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers", + "Efficient-Large-Model/Sana_1600M_1024px_diffusers", + "Efficient-Large-Model/Sana_600M_1024px_diffusers", + "Efficient-Large-Model/Sana_1600M_512px_diffusers", + "Efficient-Large-Model/Sana_600M_512px_diffusers", + ], + model_detectors=[ + lambda hf_id: ( + "sana" in hf_id.lower() + and "sana-wm" not in hf_id.lower() + and "sana_wm" not in hf_id.lower() + and "sana-video" not in hf_id.lower() + and "sana_video" not in hf_id.lower() + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video.py index 0ebc74be7..cbf276c46 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video.py @@ -116,3 +116,19 @@ class SanaVideoPipelineConfig(PipelineConfig): def gather_latents_for_sp(self, latents, batch=None): return latents + + +def register(): + from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=SanaVideoSamplingParams, + pipeline_config_cls=SanaVideoPipelineConfig, + hf_model_paths=[ + "Efficient-Large-Model/SANA-Video_2B_480p_diffusers", + ], + model_detectors=[ + lambda hf_id: "sana-video" in hf_id.lower() or "sana_video" in hf_id.lower() + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py index 1b99d00e9..3dc674f6c 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_wm.py @@ -339,3 +339,20 @@ class SanaWMRealtimeConfig(SanaWMPipelineConfig): keep_resident_components=("dit",), auto_enable_cfg_parallel=False, ) + + +def register(): + from sglang.multimodal_gen.configs.sample.sana_wm import SanaWMSamplingParams + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=SanaWMSamplingParams, + pipeline_config_cls=SanaWMPipelineConfig, + hf_model_paths=[ + "Efficient-Large-Model/SANA-WM_bidirectional", + "Efficient-Large-Model/SANA-WM_streaming", + ], + model_detectors=[ + lambda hf_id: "sana-wm" in hf_id.lower() or "sana_wm" in hf_id.lower(), + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sensenova_u1.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sensenova_u1.py index 6a5cf96c2..c94b7836f 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/sensenova_u1.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sensenova_u1.py @@ -197,3 +197,18 @@ class SenseNovaU1PipelineConfig(PipelineConfig): auto_enable_cfg_parallel=False, supports_cfg_parallel=False, ) + + +def register(): + from sglang.multimodal_gen.configs.sample.sensenova_u1 import ( + SenseNovaU1SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=SenseNovaU1SamplingParams, + pipeline_config_cls=SenseNovaU1PipelineConfig, + hf_model_paths=[ + "sensenova/SenseNova-U1.5-8B-MoT", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/stablediffusion3.py b/python/sglang/multimodal_gen/configs/pipeline_configs/stablediffusion3.py index beea21097..a972774c4 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/stablediffusion3.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/stablediffusion3.py @@ -200,3 +200,33 @@ class StableDiffusion3PipelineConfig(SpatialImagePipelineConfig): batch.height // spatial_ratio, batch.width // spatial_ratio, ) + + +def register(): + from sglang.multimodal_gen.configs.sample.stablediffusion3 import ( + StableDiffusion3SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=StableDiffusion3SamplingParams, + pipeline_config_cls=StableDiffusion3PipelineConfig, + hf_model_paths=[ + "stabilityai/stable-diffusion-3-medium", + "stabilityai/stable-diffusion-3-medium-diffusers", + "stabilityai/stable-diffusion-3.5-medium", + "stabilityai/stable-diffusion-3.5-medium-diffusers", + "stabilityai/stable-diffusion-3.5-large", + "stabilityai/stable-diffusion-3.5-large-diffusers", + ], + model_detectors=[ + lambda hf_id: ( + "stable-diffusion-3-medium" in hf_id.lower() + or "stable-diffusion-3.5-medium" in hf_id.lower() + or "stable-diffusion-3.5-large" in hf_id.lower() + or "sd3-medium" in hf_id.lower() + or "sd3.5-medium" in hf_id.lower() + or "sd3.5-large" in hf_id.lower() + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py index 584fe7269..ddcd00fee 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py @@ -312,3 +312,114 @@ class SelfForcingWanT2V480PConfig(WanT2V480PConfig): default_factory=lambda: [1000, 750, 500, 250] ) warp_denoising_step: bool = True + + +def register(): + from sglang.multimodal_gen.configs.sample.wan import ( + FastWanT2V480PConfig, + Turbo_Wan2_2_I2V_A14B_SamplingParam, + Wan2_1_Fun_1_3B_InP_SamplingParams, + Wan2_2_I2V_A14B_SamplingParam, + Wan2_2_T2V_A14B_SamplingParam, + Wan2_2_TI2V_5B_SamplingParam, + WanI2V_14B_480P_SamplingParam, + WanI2V_14B_720P_SamplingParam, + WanT2V_1_3B_SamplingParams, + WanT2V_14B_SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=WanT2V_1_3B_SamplingParams, + pipeline_config_cls=WanT2V480PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", + ], + model_detectors=[lambda hf_id: "wanpipeline" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=WanT2V_1_3B_SamplingParams, + pipeline_config_cls=TurboWanT2V1_3B480PConfig, + hf_model_paths=[ + "IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers", + ], + ) + register_configs( + sampling_param_cls=WanT2V_14B_SamplingParams, + pipeline_config_cls=WanT2V720PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-T2V-14B-Diffusers", + ], + ) + register_configs( + sampling_param_cls=WanT2V_14B_SamplingParams, + pipeline_config_cls=TurboWanT2V480PConfig, + hf_model_paths=[ + "IPostYellow/TurboWan2.1-T2V-14B-Diffusers", + "IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers", + ], + ) + register_configs( + sampling_param_cls=WanI2V_14B_480P_SamplingParam, + pipeline_config_cls=WanI2V480PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", + ], + model_detectors=[lambda hf_id: "wanimagetovideo" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=WanI2V_14B_720P_SamplingParam, + pipeline_config_cls=WanI2V720PConfig, + hf_model_paths=[ + "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Turbo_Wan2_2_I2V_A14B_SamplingParam, + pipeline_config_cls=TurboWanI2V720Config, + hf_model_paths=[ + "IPostYellow/TurboWan2.2-I2V-A14B-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParams, + pipeline_config_cls=WanI2V480PConfig, + hf_model_paths=[ + "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam, + pipeline_config_cls=Wan2_2_TI2V_5B_Config, + hf_model_paths=[ + "Wan-AI/Wan2.2-TI2V-5B-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam, + pipeline_config_cls=FastWan2_2_TI2V_5B_Config, + hf_model_paths=[ + "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", + "FastVideo/FastWan2.2-TI2V-5B-Diffusers", + ], + ) + register_configs( + sampling_param_cls=Wan2_2_T2V_A14B_SamplingParam, + pipeline_config_cls=Wan2_2_T2V_A14B_Config, + hf_model_paths=[ + "Wan-AI/Wan2.2-T2V-A14B-Diffusers", + "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4", + ], + ) + register_configs( + sampling_param_cls=Wan2_2_I2V_A14B_SamplingParam, + pipeline_config_cls=Wan2_2_I2V_A14B_Config, + hf_model_paths=["Wan-AI/Wan2.2-I2V-A14B-Diffusers"], + ) + register_configs( + sampling_param_cls=FastWanT2V480PConfig, + pipeline_config_cls=FastWan2_1_T2V_480P_Config, + hf_model_paths=[ + "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", + ], + ) diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py b/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py index ecee1a99e..f39e846a5 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/zimage.py @@ -515,3 +515,30 @@ class ZImagePipelineConfig(ZImageRolloutPipelineMixin, ImagePipelineConfig): dtype=torch.long, ), } + + +def register(): + from sglang.multimodal_gen.configs.sample.zimage import ( + ZImageSamplingParams, + ZImageTurboSamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=ZImageTurboSamplingParams, + pipeline_config_cls=ZImagePipelineConfig, + hf_model_paths=[ + "Tongyi-MAI/Z-Image-Turbo", + ], + model_detectors=[lambda hf_id: "z-image-turbo" in hf_id.lower()], + ) + register_configs( + sampling_param_cls=ZImageSamplingParams, + pipeline_config_cls=ZImagePipelineConfig, + hf_model_paths=[ + "Tongyi-MAI/Z-Image", + ], + model_detectors=[ + lambda hf_id: "z-image" in hf_id.lower() and "turbo" not in hf_id.lower() + ], + ) diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index c25d1788b..40abb10ce 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -27,193 +27,7 @@ from typing import ( if TYPE_CHECKING: from sglang.multimodal_gen.runtime.server_args import Backend -from sglang.multimodal_gen.configs.pipeline_configs import ( - Cosmos3Config, - FastH3PipelineConfig, - FastHunyuanConfig, - FluxPipelineConfig, - HeliosDistilledConfig, - HeliosMidConfig, - HeliosT2VConfig, - HunyuanConfig, - LingBotWorldCausalDMDConfig, - LingBotWorldV2CausalDMDConfig, - MiniMaxH3PipelineConfig, - WanI2V480PConfig, - WanI2V720PConfig, - WanT2V480PConfig, - WanT2V720PConfig, - ZImagePipelineConfig, -) from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig -from sglang.multimodal_gen.configs.pipeline_configs.ernie_image import ( - ErnieImagePipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.flux import ( - Flux2KleinBasePipelineConfig, - Flux2KleinPipelineConfig, - Flux2PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.glm_image import ( - GlmImagePipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import ( - Hunyuan3D2PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.ideogram import ( - Ideogram4DistilledPipelineConfig, - Ideogram4PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.joy_echo import ( - JoyEchoPipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.joy_image import ( - JoyImageEditPipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.krea2 import Krea2PipelineConfig -from sglang.multimodal_gen.configs.pipeline_configs.lingbot_video_moe import ( - LingBotVideoMoEPipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import ( - LongCatImageEditPipelineConfig, - LongCatImagePipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.longlive2 import LongLive2T2VConfig -from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import ( - LTX2PipelineConfig, - LTX23PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.ltx_2_5 import LTX25PipelineConfig -from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3_vdn import ( - VDNH3PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.mova import ( - MOVA360PConfig, - MOVA720PConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig -from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( - QwenImageEditPipelineConfig, - QwenImageEditPlus_2511_PipelineConfig, - QwenImageEditPlusPipelineConfig, - QwenImageLayeredPipelineConfig, - QwenImagePipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.qwen_image21 import ( - QwenImage21PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.sana import SanaPipelineConfig -from sglang.multimodal_gen.configs.pipeline_configs.sana_video import ( - SanaVideoPipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.sana_wm import SanaWMPipelineConfig -from sglang.multimodal_gen.configs.pipeline_configs.sensenova_u1 import ( - SenseNovaU1PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.stablediffusion3 import ( - StableDiffusion3PipelineConfig, -) -from sglang.multimodal_gen.configs.pipeline_configs.wan import ( - FastWan2_1_T2V_480P_Config, - FastWan2_2_TI2V_5B_Config, - TurboWanI2V720Config, - TurboWanT2V1_3B480PConfig, - TurboWanT2V480PConfig, - Wan2_2_I2V_A14B_Config, - Wan2_2_T2V_A14B_Config, - Wan2_2_TI2V_5B_Config, -) -from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams -from sglang.multimodal_gen.configs.sample.ernie_image import ErnieImageSamplingParams -from sglang.multimodal_gen.configs.sample.flux import ( - Flux2KleinBaseSamplingParams, - Flux2KleinSamplingParams, - Flux2SamplingParams, - FluxSamplingParams, -) -from sglang.multimodal_gen.configs.sample.glmimage import GlmImageSamplingParams -from sglang.multimodal_gen.configs.sample.helios import ( - HeliosDistilledSamplingParams, - HeliosMidSamplingParams, - HeliosT2VSamplingParams, -) -from sglang.multimodal_gen.configs.sample.hunyuan import ( - FastHunyuanSamplingParam, - HunyuanSamplingParams, -) -from sglang.multimodal_gen.configs.sample.hunyuan3d import Hunyuan3DSamplingParams -from sglang.multimodal_gen.configs.sample.ideogram import ( - Ideogram4FastSamplingParams, - Ideogram4InstantSamplingParams, - Ideogram4SamplingParams, -) -from sglang.multimodal_gen.configs.sample.joy_echo import JoyEchoSamplingParams -from sglang.multimodal_gen.configs.sample.joy_image import ( - JoyImageEditSamplingParams, -) -from sglang.multimodal_gen.configs.sample.krea2 import ( - Krea2SamplingParams, -) -from sglang.multimodal_gen.configs.sample.lingbot_video_moe import ( - LingBotVideoMoESamplingParams, -) -from sglang.multimodal_gen.configs.sample.lingbot_world import ( - LingBotWorldSamplingParams, -) -from sglang.multimodal_gen.configs.sample.longcat_image import ( - LongCatImageEditSamplingParams, - LongCatImageEditTurboSamplingParams, - LongCatImageSamplingParams, -) -from sglang.multimodal_gen.configs.sample.longlive2 import LongLive2SamplingParams -from sglang.multimodal_gen.configs.sample.ltx_2 import ( - LTX2SamplingParams, - LTX23HQSamplingParams, - LTX23SamplingParams, -) -from sglang.multimodal_gen.configs.sample.ltx_2_5 import LTX25SamplingParams -from sglang.multimodal_gen.configs.sample.minimax_h3 import ( - FastH3SamplingParams, - MiniMaxH3SamplingParams, -) -from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import VDNH3SamplingParams -from sglang.multimodal_gen.configs.sample.mova import ( - MOVA_360P_SamplingParams, - MOVA_720P_SamplingParams, -) -from sglang.multimodal_gen.configs.sample.pi05 import Pi05SamplingParams -from sglang.multimodal_gen.configs.sample.qwenimage import ( - QwenImage2512SamplingParams, - QwenImageEditPlusSamplingParams, - QwenImageLayeredSamplingParams, - QwenImageSamplingParams, -) -from sglang.multimodal_gen.configs.sample.qwenimage21 import QwenImage21SamplingParams -from sglang.multimodal_gen.configs.sample.sana import SanaSamplingParams -from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams -from sglang.multimodal_gen.configs.sample.sana_wm import SanaWMSamplingParams -from sglang.multimodal_gen.configs.sample.sensenova_u1 import ( - SenseNovaU1SamplingParams, -) -from sglang.multimodal_gen.configs.sample.stablediffusion3 import ( - StableDiffusion3SamplingParams, -) -from sglang.multimodal_gen.configs.sample.wan import ( - FastWanT2V480PConfig, - Turbo_Wan2_2_I2V_A14B_SamplingParam, - Wan2_1_Fun_1_3B_InP_SamplingParams, - Wan2_2_I2V_A14B_SamplingParam, - Wan2_2_T2V_A14B_SamplingParam, - Wan2_2_TI2V_5B_SamplingParam, - WanI2V_14B_480P_SamplingParam, - WanI2V_14B_720P_SamplingParam, - WanT2V_1_3B_SamplingParams, - WanT2V_14B_SamplingParams, -) -from sglang.multimodal_gen.configs.sample.zimage import ( - ZImageSamplingParams, - ZImageTurboSamplingParams, -) from sglang.multimodal_gen.configs.sensenova_u1 import ( SENSENOVA_U1_MODEL_IDS, is_sensenova_u1_adapter_only_model, @@ -375,6 +189,9 @@ def register_configs( pipeline_config_cls: Type[PipelineConfig], hf_model_paths: Optional[List[str]] = None, model_detectors: Optional[List[Callable[[str], bool]]] = None, + pipeline_config_registry_entries: Optional[ + Dict[str, Tuple[Type[PipelineConfig], Type[Any]]] + ] = None, ) -> str: """ Registers configuration classes for a new model family. @@ -396,6 +213,11 @@ def register_configs( if model_detectors: for detector in model_detectors: _MODEL_NAME_DETECTORS.append((model_id, detector)) + + if pipeline_config_registry_entries: + for pipeline_name, (pc_cls, sp_cls) in pipeline_config_registry_entries.items(): + _PIPELINE_CONFIG_REGISTRY.setdefault(pipeline_name, (pc_cls, sp_cls)) + return model_id @@ -459,6 +281,56 @@ def register_pipeline( ) +_configs_discovered: bool = False + +# SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match) +# SANA-Video (register before generic SANA to avoid detector overlap). +_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video") + + +def _discover_and_register_configs() -> None: + global _configs_discovered + if _configs_discovered: + return + _configs_discovered = True + + package_name = "sglang.multimodal_gen.configs.pipeline_configs" + package = importlib.import_module(package_name) + + discovered = [] + for _, module_name, ispkg in pkgutil.walk_packages( + package.__path__, package.__name__ + "." + ): + if not ispkg: + try: + config_module = importlib.import_module(module_name) + except Exception as exc: + logger.warning( + f"Skipping config module {module_name} during discovery due to import failure: {exc}", + ) + continue + if hasattr(config_module, "register"): + discovered.append((module_name, config_module)) + + def _sort_key(item): + short_name = item[0].rsplit(".", 1)[-1] + try: + return (0, _CONFIG_REGISTER_PRIORITY.index(short_name)) + except ValueError: + return (1, 0) + + discovered.sort(key=_sort_key) + + for module_name, config_module in discovered: + try: + config_module.register() + except Exception as exc: + logger.warning( + f"register() failed for {module_name}: {exc}", + exc_info=True, + ) + + def get_model_short_name(model_id: str) -> str: if "/" in model_id: return model_id.rstrip("/").split("/")[-1] @@ -809,657 +681,7 @@ def get_model_info( return model_info -# Registration of model configs -def _register_configs(): - # Pi0.5 / OpenPI / LeRobot action policies. - register_configs( - sampling_param_cls=Pi05SamplingParams, - pipeline_config_cls=Pi05PipelineConfig, - hf_model_paths=[ - "lerobot/pi05_base", - "lerobot/pi05_libero_base", - ], - model_detectors=[ - lambda hf_id: "pi05" in hf_id.lower(), - lambda hf_id: "pi0.5" in hf_id.lower(), - ], - ) - - # LTX-2 - register_configs( - sampling_param_cls=LTX2SamplingParams, - pipeline_config_cls=LTX2PipelineConfig, - hf_model_paths=["Lightricks/LTX-2"], - model_detectors=[ - lambda path: "ltx" in path.lower() and "video" in path.lower(), - lambda path: ( - "ltx-2" in path.lower() - and "ltx-2.3" not in path.lower() - and "ltx-2.5" not in path.lower() - ), - ], - ) - register_configs( - sampling_param_cls=LTX23SamplingParams, - pipeline_config_cls=LTX23PipelineConfig, - hf_model_paths=["Lightricks/LTX-2.3"], - model_detectors=[ - lambda path: "ltx-2.3" in path.lower(), - ], - ) - # Keeps the LTX-2 pipeline class; only component geometry and the pinned - # distilled schedule differ. Only the `-Diffusers` repo is listed -- - # `Lightricks/LTX-2.5` is a split pack of bare `.safetensors` and would need - # a model overlay first. - register_configs( - sampling_param_cls=LTX25SamplingParams, - pipeline_config_cls=LTX25PipelineConfig, - hf_model_paths=["Lightricks/LTX-2.5-Diffusers"], - model_detectors=[ - lambda path: "ltx-2.5" in path.lower(), - ], - ) - # register dedicated sampling params for LTX2TwoStageHQPipeline - _PIPELINE_CONFIG_REGISTRY.setdefault( - "LTX2TwoStageHQPipeline", - (LTX2PipelineConfig, LTX23HQSamplingParams), - ) - - # Hunyuan - register_configs( - sampling_param_cls=HunyuanSamplingParams, - pipeline_config_cls=HunyuanConfig, - hf_model_paths=[ - "hunyuanvideo-community/HunyuanVideo", - ], - model_detectors=[lambda hf_id: "hunyuanvideo" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=FastHunyuanSamplingParam, - pipeline_config_cls=FastHunyuanConfig, - hf_model_paths=[ - "FastVideo/FastHunyuan-diffusers", - ], - ) - # Wan - register_configs( - sampling_param_cls=WanT2V_1_3B_SamplingParams, - pipeline_config_cls=WanT2V480PConfig, - hf_model_paths=[ - "Wan-AI/Wan2.1-T2V-1.3B-Diffusers", - ], - model_detectors=[lambda hf_id: "wanpipeline" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=WanT2V_1_3B_SamplingParams, - pipeline_config_cls=TurboWanT2V1_3B480PConfig, - hf_model_paths=[ - "IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers", - ], - ) - register_configs( - sampling_param_cls=WanT2V_14B_SamplingParams, - pipeline_config_cls=WanT2V720PConfig, - hf_model_paths=[ - "Wan-AI/Wan2.1-T2V-14B-Diffusers", - ], - ) - register_configs( - sampling_param_cls=WanT2V_14B_SamplingParams, - pipeline_config_cls=TurboWanT2V480PConfig, - hf_model_paths=[ - "IPostYellow/TurboWan2.1-T2V-14B-Diffusers", - "IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers", - ], - ) - register_configs( - sampling_param_cls=WanI2V_14B_480P_SamplingParam, - pipeline_config_cls=WanI2V480PConfig, - hf_model_paths=[ - "Wan-AI/Wan2.1-I2V-14B-480P-Diffusers", - ], - model_detectors=[lambda hf_id: "wanimagetovideo" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=WanI2V_14B_720P_SamplingParam, - pipeline_config_cls=WanI2V720PConfig, - hf_model_paths=[ - "Wan-AI/Wan2.1-I2V-14B-720P-Diffusers", - ], - ) - register_configs( - sampling_param_cls=Turbo_Wan2_2_I2V_A14B_SamplingParam, - pipeline_config_cls=TurboWanI2V720Config, - hf_model_paths=[ - "IPostYellow/TurboWan2.2-I2V-A14B-Diffusers", - ], - ) - register_configs( - sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParams, - pipeline_config_cls=WanI2V480PConfig, - hf_model_paths=[ - "weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers", - ], - ) - register_configs( - sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam, - pipeline_config_cls=Wan2_2_TI2V_5B_Config, - hf_model_paths=[ - "Wan-AI/Wan2.2-TI2V-5B-Diffusers", - ], - ) - register_configs( - sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam, - pipeline_config_cls=FastWan2_2_TI2V_5B_Config, - hf_model_paths=[ - "FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers", - "FastVideo/FastWan2.2-TI2V-5B-Diffusers", - ], - ) - register_configs( - sampling_param_cls=Wan2_2_T2V_A14B_SamplingParam, - pipeline_config_cls=Wan2_2_T2V_A14B_Config, - hf_model_paths=[ - "Wan-AI/Wan2.2-T2V-A14B-Diffusers", - "nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4", - ], - ) - register_configs( - sampling_param_cls=Wan2_2_I2V_A14B_SamplingParam, - pipeline_config_cls=Wan2_2_I2V_A14B_Config, - hf_model_paths=["Wan-AI/Wan2.2-I2V-A14B-Diffusers"], - ) - register_configs( - sampling_param_cls=LingBotWorldSamplingParams, - pipeline_config_cls=LingBotWorldCausalDMDConfig, - hf_model_paths=[ - "IPostYellow/lingbot-world-fast-diffusers", - "robbyant/lingbot-world-fast-diffusers", - ], - ) - register_configs( - sampling_param_cls=LingBotWorldSamplingParams, - pipeline_config_cls=LingBotWorldV2CausalDMDConfig, - hf_model_paths=[ - "robbyant/lingbot-world-v2-14b-causal-fast-diffusers", - ], - ) - register_configs( - sampling_param_cls=LongLive2SamplingParams, - pipeline_config_cls=LongLive2T2VConfig, - hf_model_paths=[ - # Since LongLive-2.0-5B does not have official diffusers release - "Rabinovich/LongLive-2.0-5B-Diffusers", - "Efficient-Large-Model/LongLive-2.0-5B", - ], - ) - register_configs( - sampling_param_cls=FastWanT2V480PConfig, - pipeline_config_cls=FastWan2_1_T2V_480P_Config, - hf_model_paths=[ - "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", - ], - ) - # MOVA - register_configs( - sampling_param_cls=MOVA_360P_SamplingParams, - pipeline_config_cls=MOVA360PConfig, - model_detectors=[ - lambda hf_id: "mova" in hf_id.lower() and "360p" in hf_id.lower() - ], - ) - register_configs( - sampling_param_cls=MOVA_720P_SamplingParams, - pipeline_config_cls=MOVA720PConfig, - model_detectors=[ - lambda hf_id: "mova" in hf_id.lower() and "720p" in hf_id.lower() - ], - ) - register_configs( - sampling_param_cls=MiniMaxH3SamplingParams, - pipeline_config_cls=MiniMaxH3PipelineConfig, - hf_model_paths=[ - "MiniMaxAI/MiniMax-H3", - "MiniMax/MiniMax-H3", - ], - model_detectors=[ - lambda model_id: ( - "minimaxh3" in model_id.lower().replace("-", "").replace("_", "") - and "vdn" not in model_id.lower() - ) - ], - ) - register_configs( - sampling_param_cls=SenseNovaU1SamplingParams, - pipeline_config_cls=SenseNovaU1PipelineConfig, - hf_model_paths=[ - "sensenova/SenseNova-U1.5-8B-MoT", - ], - ) - register_configs( - sampling_param_cls=FastH3SamplingParams, - pipeline_config_cls=FastH3PipelineConfig, - hf_model_paths=[ - "FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree", - ], - model_detectors=[ - lambda model_id: ( - "fasth3" in model_id.lower().replace("-", "").replace("_", "") - ) - ], - ) - register_configs( - sampling_param_cls=VDNH3SamplingParams, - pipeline_config_cls=VDNH3PipelineConfig, - hf_model_paths=[ - "OpenVDN/vdn-minimax-h3", - ], - model_detectors=[ - lambda model_id: ( - "vdn" in model_id.lower() - and "minimaxh3" in model_id.lower().replace("-", "").replace("_", "") - ) - ], - ) - # FLUX - register_configs( - sampling_param_cls=FluxSamplingParams, - pipeline_config_cls=FluxPipelineConfig, - hf_model_paths=[ - "black-forest-labs/FLUX.1-dev", - ], - model_detectors=[lambda hf_id: "flux.1" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=Flux2KleinSamplingParams, - pipeline_config_cls=Flux2KleinPipelineConfig, - hf_model_paths=[ - "black-forest-labs/FLUX.2-klein-4B", - "black-forest-labs/FLUX.2-klein-9B", - ], - model_detectors=[ - lambda hf_id: ( - ("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower()) - and "base" not in hf_id.lower() - ) - ], - ) - register_configs( - sampling_param_cls=Flux2KleinBaseSamplingParams, - pipeline_config_cls=Flux2KleinBasePipelineConfig, - hf_model_paths=[ - "black-forest-labs/FLUX.2-klein-base-4B", - "black-forest-labs/FLUX.2-klein-base-9B", - ], - model_detectors=[ - lambda hf_id: ( - ("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower()) - and "base" in hf_id.lower() - ) - ], - ) - register_configs( - sampling_param_cls=Flux2SamplingParams, - pipeline_config_cls=Flux2PipelineConfig, - hf_model_paths=[ - "black-forest-labs/FLUX.2-dev", - "black-forest-labs/FLUX.2-dev-NVFP4", - ], - model_detectors=[ - lambda hf_id: "flux.2" in hf_id.lower() and "klein" not in hf_id.lower() - ], - ) - register_configs( - sampling_param_cls=ZImageTurboSamplingParams, - pipeline_config_cls=ZImagePipelineConfig, - hf_model_paths=[ - "Tongyi-MAI/Z-Image-Turbo", - ], - model_detectors=[lambda hf_id: "z-image-turbo" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=ZImageSamplingParams, - pipeline_config_cls=ZImagePipelineConfig, - hf_model_paths=[ - "Tongyi-MAI/Z-Image", - ], - model_detectors=[ - lambda hf_id: "z-image" in hf_id.lower() and "turbo" not in hf_id.lower() - ], - ) - # Krea-2 (K2) - register_configs( - sampling_param_cls=Krea2SamplingParams, - pipeline_config_cls=Krea2PipelineConfig, - hf_model_paths=["krea/Krea-2"], - model_detectors=[lambda hf_id: "krea-2" in hf_id.lower()], - ) - # Qwen-Image - register_configs( - sampling_param_cls=QwenImage21SamplingParams, - pipeline_config_cls=QwenImage21PipelineConfig, - hf_model_paths=["Qwen/Qwen-Image-2.1"], - model_detectors=[lambda hf_id: "qwen-image-2.1" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=QwenImageSamplingParams, - pipeline_config_cls=QwenImagePipelineConfig, - hf_model_paths=["Qwen/Qwen-Image", "nvidia/Qwen-Image-NVFP4"], - model_detectors=[ - lambda hf_id: ( - "qwen-image" in hf_id.lower() - and "edit" not in hf_id.lower() - and "layered" not in hf_id.lower() - and "2512" not in hf_id.lower() - and "qwen-image-2.1" not in hf_id.lower() - ) - ], - ) - register_configs( - sampling_param_cls=QwenImage2512SamplingParams, - pipeline_config_cls=QwenImagePipelineConfig, - hf_model_paths=["Qwen/Qwen-Image-2512"], - model_detectors=[lambda hf_id: "qwen-image-2512" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=QwenImageSamplingParams, - pipeline_config_cls=QwenImageEditPipelineConfig, - hf_model_paths=["Qwen/Qwen-Image-Edit"], - model_detectors=[ - lambda hf_id: ( - "qwen-image-edit" in hf_id.lower() - and "2509" not in hf_id.lower() - and "2511" not in hf_id.lower() - ) - ], - ) - - register_configs( - sampling_param_cls=QwenImageEditPlusSamplingParams, - pipeline_config_cls=QwenImageEditPlusPipelineConfig, - hf_model_paths=["Qwen/Qwen-Image-Edit-2509"], - model_detectors=[lambda hf_id: "qwen-image-edit-2509" in hf_id.lower()], - ) - - register_configs( - sampling_param_cls=QwenImageEditPlusSamplingParams, - pipeline_config_cls=QwenImageEditPlus_2511_PipelineConfig, - hf_model_paths=["Qwen/Qwen-Image-Edit-2511"], - model_detectors=[lambda hf_id: "qwen-image-edit-2511" in hf_id.lower()], - ) - - register_configs( - sampling_param_cls=QwenImageLayeredSamplingParams, - pipeline_config_cls=QwenImageLayeredPipelineConfig, - hf_model_paths=["Qwen/Qwen-Image-Layered"], - model_detectors=[lambda hf_id: "qwen-image-layered" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=StableDiffusion3SamplingParams, - pipeline_config_cls=StableDiffusion3PipelineConfig, - hf_model_paths=[ - "stabilityai/stable-diffusion-3-medium", - "stabilityai/stable-diffusion-3-medium-diffusers", - "stabilityai/stable-diffusion-3.5-medium", - "stabilityai/stable-diffusion-3.5-medium-diffusers", - "stabilityai/stable-diffusion-3.5-large", - "stabilityai/stable-diffusion-3.5-large-diffusers", - ], - model_detectors=[ - lambda hf_id: ( - "stable-diffusion-3-medium" in hf_id.lower() - or "stable-diffusion-3.5-medium" in hf_id.lower() - or "stable-diffusion-3.5-large" in hf_id.lower() - or "sd3-medium" in hf_id.lower() - or "sd3.5-medium" in hf_id.lower() - or "sd3.5-large" in hf_id.lower() - ) - ], - ) - - register_configs( - sampling_param_cls=GlmImageSamplingParams, - pipeline_config_cls=GlmImagePipelineConfig, - model_detectors=[lambda hf_id: "glm-image" in hf_id.lower()], - ) - register_configs( - sampling_param_cls=Hunyuan3DSamplingParams, - pipeline_config_cls=Hunyuan3D2PipelineConfig, - hf_model_paths=[ - "tencent/Hunyuan3D-2", - ], - model_detectors=[lambda hf_id: "hunyuan3d" in hf_id.lower()], - ) - - # Helios - register_configs( - sampling_param_cls=HeliosT2VSamplingParams, - pipeline_config_cls=HeliosT2VConfig, - hf_model_paths=[ - "BestWishYsh/Helios-Base", - ], - model_detectors=[ - lambda hf_id: ( - "helios" in hf_id.lower() - and "mid" not in hf_id.lower() - and "distill" not in hf_id.lower() - ) - ], - ) - register_configs( - sampling_param_cls=HeliosMidSamplingParams, - pipeline_config_cls=HeliosMidConfig, - hf_model_paths=[ - "BestWishYsh/Helios-Mid", - ], - ) - register_configs( - sampling_param_cls=HeliosDistilledSamplingParams, - pipeline_config_cls=HeliosDistilledConfig, - hf_model_paths=[ - "BestWishYsh/Helios-Distilled", - ], - ) - - # SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match) - register_configs( - sampling_param_cls=SanaWMSamplingParams, - pipeline_config_cls=SanaWMPipelineConfig, - hf_model_paths=[ - "Efficient-Large-Model/SANA-WM_bidirectional", - "Efficient-Large-Model/SANA-WM_streaming", - ], - model_detectors=[ - # Match "sana-wm" or "sana_wm" but NOT plain T2I "sana" checkpoints. - lambda hf_id: "sana-wm" in hf_id.lower() or "sana_wm" in hf_id.lower(), - ], - ) - - # SANA-Video (register before generic SANA to avoid detector overlap). - register_configs( - sampling_param_cls=SanaVideoSamplingParams, - pipeline_config_cls=SanaVideoPipelineConfig, - hf_model_paths=[ - "Efficient-Large-Model/SANA-Video_2B_480p_diffusers", - ], - model_detectors=[ - lambda hf_id: "sana-video" in hf_id.lower() or "sana_video" in hf_id.lower() - ], - ) - - # Cosmos3 — single checkpoint serves T2V, I2V, and T2I. Mode is dispatched - # per-request inside the pipeline from ``num_frames`` and ``image_path``. - # All variants share the same pipeline; arch dimensions (size, activation, - # QK-norm) come from ``transformer/config.json`` via ``update_model_arch``. - register_configs( - sampling_param_cls=Cosmos3SamplingParams, - pipeline_config_cls=Cosmos3Config, - hf_model_paths=[ - "nvidia/Cosmos3-Nano", - "nvidia/Cosmos3-Nano-Policy-DROID", - "nvidia/Cosmos3-Super", - "nvidia/Cosmos3-Super-Text2Image", - "nvidia/Cosmos3-Super-Image2Video", - "nvidia/Cosmos3-Edge", - ], - # Match both the new ``Cosmos3OmniPipeline`` and the legacy - # ``Cosmos3OmniDiffusersPipeline`` ``_class_name`` (diffusers rename). - model_detectors=[lambda hf_id: "cosmos3omni" in hf_id.lower()], - ) - - # SANA - register_configs( - sampling_param_cls=SanaSamplingParams, - pipeline_config_cls=SanaPipelineConfig, - hf_model_paths=[ - "Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers", - "Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers", - "Efficient-Large-Model/Sana_1600M_1024px_diffusers", - "Efficient-Large-Model/Sana_600M_1024px_diffusers", - "Efficient-Large-Model/Sana_1600M_512px_diffusers", - "Efficient-Large-Model/Sana_600M_512px_diffusers", - ], - model_detectors=[ - lambda hf_id: ( - "sana" in hf_id.lower() - and "sana-wm" not in hf_id.lower() - and "sana_wm" not in hf_id.lower() - and "sana-video" not in hf_id.lower() - and "sana_video" not in hf_id.lower() - ) - ], - ) - - # FireRed-Image-Edit - register_configs( - sampling_param_cls=QwenImageEditPlusSamplingParams, - pipeline_config_cls=QwenImageEditPlusPipelineConfig, - hf_model_paths=[ - "FireRedTeam/FireRed-Image-Edit-1.0", - "FireRedTeam/FireRed-Image-Edit-1.1", - ], - ) - - # ErnieImage - register_configs( - sampling_param_cls=ErnieImageSamplingParams, - pipeline_config_cls=ErnieImagePipelineConfig, - hf_model_paths=[ - "baidu/ERNIE-Image", - "baidu/ERNIE-Image-Turbo", - ], - model_detectors=[ - lambda hf_id: "ernie-image" in hf_id.lower(), - ], - ) - - # JoyAI - register_configs( - sampling_param_cls=JoyImageEditSamplingParams, - pipeline_config_cls=JoyImageEditPipelineConfig, - hf_model_paths=[ - "jdopensource/JoyAI-Image-Edit-Diffusers", - ], - model_detectors=[ - lambda hf_id: "joyai-image-edit" in hf_id.lower(), - ], - ) - register_configs( - sampling_param_cls=JoyEchoSamplingParams, - pipeline_config_cls=JoyEchoPipelineConfig, - hf_model_paths=[ - "jdopensource/JoyAI-Echo", - ], - model_detectors=[ - lambda hf_id: ( - ("joy-echo" in hf_id.lower() or "joyai-echo" in hf_id.lower()) - and "image-edit" not in hf_id.lower() - ), - ], - ) - - # Ideogram 4 - register_configs( - sampling_param_cls=Ideogram4FastSamplingParams, - pipeline_config_cls=Ideogram4DistilledPipelineConfig, - hf_model_paths=["fal/ideogram-v4-fast"], - ) - register_configs( - sampling_param_cls=Ideogram4InstantSamplingParams, - pipeline_config_cls=Ideogram4DistilledPipelineConfig, - hf_model_paths=["fal/ideogram-v4-instant"], - ) - register_configs( - sampling_param_cls=Ideogram4SamplingParams, - pipeline_config_cls=Ideogram4PipelineConfig, - hf_model_paths=[ - "ideogram-ai/ideogram-4-fp8", - "ideogram-ai/ideogram-4-nf4", - "Comfy-Org/Ideogram-4", - ], - model_detectors=[ - lambda hf_id: "ideogram4pipeline" in hf_id.lower(), - lambda hf_id: "ideogram-4-fp8" in hf_id.lower(), - lambda hf_id: "ideogram-4-nf4" in hf_id.lower(), - lambda hf_id: "comfy-org/ideogram-4" in hf_id.lower(), - lambda hf_id: "comfy-org--ideogram-4" in hf_id.lower(), - ], - ) - - register_configs( - sampling_param_cls=LingBotVideoMoESamplingParams, - pipeline_config_cls=LingBotVideoMoEPipelineConfig, - model_detectors=[ - lambda hf_id: "lingbot-video-moe" in hf_id.lower(), - ], - ) - - # LongCat-Image - register_configs( - sampling_param_cls=LongCatImageSamplingParams, - pipeline_config_cls=LongCatImagePipelineConfig, - hf_model_paths=[ - "meituan-longcat/LongCat-Image", - ], - model_detectors=[ - lambda hf_id: "longcat" in hf_id.lower() and "edit" not in hf_id.lower(), - ], - ) - - # LongCat-Image-Edit-Turbo (registered before Edit so its detector wins) - register_configs( - sampling_param_cls=LongCatImageEditTurboSamplingParams, - pipeline_config_cls=LongCatImageEditPipelineConfig, - hf_model_paths=[ - "meituan-longcat/LongCat-Image-Edit-Turbo", - ], - model_detectors=[ - lambda hf_id: ( - "longcat" in hf_id.lower() - and "edit" in hf_id.lower() - and "turbo" in hf_id.lower() - ), - ], - ) - - # LongCat-Image-Edit - register_configs( - sampling_param_cls=LongCatImageEditSamplingParams, - pipeline_config_cls=LongCatImageEditPipelineConfig, - hf_model_paths=[ - "meituan-longcat/LongCat-Image-Edit", - ], - model_detectors=[ - lambda hf_id: ( - "longcat" in hf_id.lower() - and "edit" in hf_id.lower() - and "turbo" not in hf_id.lower() - ), - ], - ) - - -_register_configs() +_discover_and_register_configs() def is_known_non_diffusers_multimodal_model(model_path: str) -> bool: