[Diffusion] migrate the whole _register_configs from registry.py to the model own config file (#40475)
This commit is contained in:
@@ -12,12 +12,11 @@ though the runtime is split into separate folders.
|
|||||||
The files are split by runtime responsibility. For a new model, read the
|
The files are split by runtime responsibility. For a new model, read the
|
||||||
request path first:
|
request path first:
|
||||||
|
|
||||||
1. `registry.py` chooses the model family, sampling params, and pipeline config.
|
1. `configs/pipeline_configs/{model}.py` defines model-specific denoising and
|
||||||
2. `configs/pipeline_configs/{model}.py` defines model-specific denoising and
|
|
||||||
decoding behavior.
|
decoding behavior.
|
||||||
3. `runtime/pipelines/{model}.py` wires modules into stages.
|
2. `runtime/pipelines/{model}.py` wires modules into stages.
|
||||||
4. `runtime/pipelines_core/stages/` runs the shared stage logic.
|
3. `runtime/pipelines_core/stages/` runs the shared stage logic.
|
||||||
5. `runtime/models/` contains native model components only when the architecture
|
4. `runtime/models/` contains native model components only when the architecture
|
||||||
cannot be reused.
|
cannot be reused.
|
||||||
|
|
||||||
That is the dependency direction. Avoid making a model PR that requires readers
|
That is the dependency direction. Avoid making a model PR that requires readers
|
||||||
@@ -124,7 +123,6 @@ behavior.
|
|||||||
|
|
||||||
| Area | Add or edit when | Typical file |
|
| Area | Add or edit when | Typical file |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Registry | Always, unless extending an already registered family | `python/sglang/multimodal_gen/registry.py` |
|
|
||||||
| Runtime parameters | The request schema differs from existing models | `configs/sample/{model}.py` |
|
| Runtime parameters | The request schema differs from existing models | `configs/sample/{model}.py` |
|
||||||
| Pipeline config | Denoising, decoding, precision, position encoding, or CFG hooks differ | `configs/pipeline_configs/{model}.py` |
|
| Pipeline config | Denoising, decoding, precision, position encoding, or CFG hooks differ | `configs/pipeline_configs/{model}.py` |
|
||||||
| Pipeline wiring | The model needs a new stage layout or module list | `runtime/pipelines/{model}.py` |
|
| Pipeline wiring | The model needs a new stage layout or module list | `runtime/pipelines/{model}.py` |
|
||||||
@@ -137,11 +135,10 @@ behavior.
|
|||||||
|
|
||||||
For a new native architecture, the common minimum is:
|
For a new native architecture, the common minimum is:
|
||||||
|
|
||||||
1. `registry.py`
|
1. `configs/sample/{model}.py`
|
||||||
2. `configs/sample/{model}.py`
|
2. `configs/pipeline_configs/{model}.py`
|
||||||
3. `configs/pipeline_configs/{model}.py`
|
3. `runtime/pipelines/{model}.py`
|
||||||
4. `runtime/pipelines/{model}.py`
|
4. `runtime/models/dits/{model}.py`
|
||||||
5. `runtime/models/dits/{model}.py`
|
|
||||||
|
|
||||||
Every extra file should map to model behavior that existing code cannot express
|
Every extra file should map to model behavior that existing code cannot express
|
||||||
clearly.
|
clearly.
|
||||||
@@ -499,19 +496,27 @@ native integration contract.
|
|||||||
|
|
||||||
### 6. Registry
|
### 6. Registry
|
||||||
|
|
||||||
Register the family once the sampling params and pipeline config exist.
|
Define a `register()` function in `configs/pipeline_configs/{model}.py`. The
|
||||||
|
runtime auto-discovers it on startup and calls it to register the sampling
|
||||||
|
params and pipeline config.
|
||||||
|
|
||||||
```python
|
```python
|
||||||
|
# python/sglang/multimodal_gen/configs/pipeline_configs/my_model.py
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
register_configs(
|
register_configs(
|
||||||
model_family="my_model",
|
|
||||||
sampling_param_cls=MyModelSamplingParams,
|
sampling_param_cls=MyModelSamplingParams,
|
||||||
pipeline_config_cls=MyModelPipelineConfig,
|
pipeline_config_cls=MyModelPipelineConfig,
|
||||||
hf_model_paths=["org/my-model"],
|
hf_model_paths=["org/my-model"],
|
||||||
|
model_detectors=[lambda hf_id: "my-model" in hf_id.lower()],
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
The pipeline file is discovered through its `EntryClass`; do not add a second
|
`model_detectors` matches a model path or `model_index.json` `_class_name` when
|
||||||
pipeline registry unless the existing registry requires it.
|
the Hugging Face path varies; see `wan.py` or `qwen_image21.py` for real
|
||||||
|
examples. The pipeline file is discovered through its `EntryClass`; do not add
|
||||||
|
a second pipeline registry unless the existing registry requires it.
|
||||||
|
|
||||||
## Verify the Port
|
## Verify the Port
|
||||||
|
|
||||||
|
|||||||
@@ -202,3 +202,24 @@ class Cosmos3Config(PipelineConfig):
|
|||||||
keep_resident_min_available_gb=threshold_gb,
|
keep_resident_min_available_gb=threshold_gb,
|
||||||
keep_resident_components=("dit", "vae"),
|
keep_resident_components=("dit", "vae"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.cosmos3 import (
|
||||||
|
Cosmos3SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Cosmos3SamplingParams,
|
||||||
|
pipeline_config_cls=Cosmos3Config,
|
||||||
|
hf_model_paths=[
|
||||||
|
"nvidia/Cosmos3-Nano",
|
||||||
|
"nvidia/Cosmos3-Nano-Policy-DROID",
|
||||||
|
"nvidia/Cosmos3-Super",
|
||||||
|
"nvidia/Cosmos3-Super-Text2Image",
|
||||||
|
"nvidia/Cosmos3-Super-Image2Video",
|
||||||
|
"nvidia/Cosmos3-Edge",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "cosmos3omni" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
|||||||
@@ -258,3 +258,22 @@ class ErnieImagePipelineConfig(ImagePipelineConfig):
|
|||||||
|
|
||||||
def post_denoising_loop(self, latents, batch):
|
def post_denoising_loop(self, latents, batch):
|
||||||
return latents
|
return latents
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.ernie_image import (
|
||||||
|
ErnieImageSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=ErnieImageSamplingParams,
|
||||||
|
pipeline_config_cls=ErnieImagePipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"baidu/ERNIE-Image",
|
||||||
|
"baidu/ERNIE-Image-Turbo",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "ernie-image" in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -823,3 +823,61 @@ class Flux2KleinBasePipelineConfig(Flux2KleinPipelineConfig):
|
|||||||
txt_seq_lens,
|
txt_seq_lens,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.flux import (
|
||||||
|
Flux2KleinBaseSamplingParams,
|
||||||
|
Flux2KleinSamplingParams,
|
||||||
|
Flux2SamplingParams,
|
||||||
|
FluxSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=FluxSamplingParams,
|
||||||
|
pipeline_config_cls=FluxPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"black-forest-labs/FLUX.1-dev",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "flux.1" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Flux2KleinSamplingParams,
|
||||||
|
pipeline_config_cls=Flux2KleinPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"black-forest-labs/FLUX.2-klein-4B",
|
||||||
|
"black-forest-labs/FLUX.2-klein-9B",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower())
|
||||||
|
and "base" not in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Flux2KleinBaseSamplingParams,
|
||||||
|
pipeline_config_cls=Flux2KleinBasePipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"black-forest-labs/FLUX.2-klein-base-4B",
|
||||||
|
"black-forest-labs/FLUX.2-klein-base-9B",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower())
|
||||||
|
and "base" in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Flux2SamplingParams,
|
||||||
|
pipeline_config_cls=Flux2PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"black-forest-labs/FLUX.2-dev",
|
||||||
|
"black-forest-labs/FLUX.2-dev-NVFP4",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "flux.2" in hf_id.lower() and "klein" not in hf_id.lower()
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -118,3 +118,16 @@ class GlmImagePipelineConfig(SpatialImagePipelineConfig):
|
|||||||
|
|
||||||
def post_decoding(self, frames, server_args):
|
def post_decoding(self, frames, server_args):
|
||||||
return self.image_processor.postprocess(frames, output_type="latent")
|
return self.image_processor.postprocess(frames, output_type="latent")
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.glmimage import (
|
||||||
|
GlmImageSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=GlmImageSamplingParams,
|
||||||
|
pipeline_config_cls=GlmImagePipelineConfig,
|
||||||
|
model_detectors=[lambda hf_id: "glm-image" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
|||||||
@@ -128,3 +128,41 @@ class HeliosDistilledConfig(HeliosT2VConfig):
|
|||||||
pyramid_num_inference_steps_list: list[int] = field(
|
pyramid_num_inference_steps_list: list[int] = field(
|
||||||
default_factory=lambda: [10, 10, 10]
|
default_factory=lambda: [10, 10, 10]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.helios import (
|
||||||
|
HeliosDistilledSamplingParams,
|
||||||
|
HeliosMidSamplingParams,
|
||||||
|
HeliosT2VSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=HeliosT2VSamplingParams,
|
||||||
|
pipeline_config_cls=HeliosT2VConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"BestWishYsh/Helios-Base",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"helios" in hf_id.lower()
|
||||||
|
and "mid" not in hf_id.lower()
|
||||||
|
and "distill" not in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=HeliosMidSamplingParams,
|
||||||
|
pipeline_config_cls=HeliosMidConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"BestWishYsh/Helios-Mid",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=HeliosDistilledSamplingParams,
|
||||||
|
pipeline_config_cls=HeliosDistilledConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"BestWishYsh/Helios-Distilled",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -169,3 +169,27 @@ class FastHunyuanConfig(HunyuanConfig):
|
|||||||
keep_resident_min_available_gb=60,
|
keep_resident_min_available_gb=60,
|
||||||
keep_resident_components=("dit", "vae"),
|
keep_resident_components=("dit", "vae"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.hunyuan import (
|
||||||
|
FastHunyuanSamplingParam,
|
||||||
|
HunyuanSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=HunyuanSamplingParams,
|
||||||
|
pipeline_config_cls=HunyuanConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"hunyuanvideo-community/HunyuanVideo",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "hunyuanvideo" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=FastHunyuanSamplingParam,
|
||||||
|
pipeline_config_cls=FastHunyuanConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"FastVideo/FastHunyuan-diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -78,3 +78,19 @@ class Hunyuan3D2PipelineConfig(PipelineConfig):
|
|||||||
latent_shape = self.vae_config.arch_config.latent_shape
|
latent_shape = self.vae_config.arch_config.latent_shape
|
||||||
shape = (batch_size, *latent_shape)
|
shape = (batch_size, *latent_shape)
|
||||||
return shape
|
return shape
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.hunyuan3d import (
|
||||||
|
Hunyuan3DSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Hunyuan3DSamplingParams,
|
||||||
|
pipeline_config_cls=Hunyuan3D2PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"tencent/Hunyuan3D-2",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "hunyuan3d" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
|||||||
@@ -308,3 +308,39 @@ class Ideogram4PipelineConfig(ImagePipelineConfig):
|
|||||||
@dataclass
|
@dataclass
|
||||||
class Ideogram4DistilledPipelineConfig(Ideogram4PipelineConfig):
|
class Ideogram4DistilledPipelineConfig(Ideogram4PipelineConfig):
|
||||||
dit_config: DiTConfig = field(default_factory=Ideogram4DistilledDiTConfig)
|
dit_config: DiTConfig = field(default_factory=Ideogram4DistilledDiTConfig)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.ideogram import (
|
||||||
|
Ideogram4FastSamplingParams,
|
||||||
|
Ideogram4InstantSamplingParams,
|
||||||
|
Ideogram4SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Ideogram4FastSamplingParams,
|
||||||
|
pipeline_config_cls=Ideogram4DistilledPipelineConfig,
|
||||||
|
hf_model_paths=["fal/ideogram-v4-fast"],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Ideogram4InstantSamplingParams,
|
||||||
|
pipeline_config_cls=Ideogram4DistilledPipelineConfig,
|
||||||
|
hf_model_paths=["fal/ideogram-v4-instant"],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Ideogram4SamplingParams,
|
||||||
|
pipeline_config_cls=Ideogram4PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"ideogram-ai/ideogram-4-fp8",
|
||||||
|
"ideogram-ai/ideogram-4-nf4",
|
||||||
|
"Comfy-Org/Ideogram-4",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "ideogram4pipeline" in hf_id.lower(),
|
||||||
|
lambda hf_id: "ideogram-4-fp8" in hf_id.lower(),
|
||||||
|
lambda hf_id: "ideogram-4-nf4" in hf_id.lower(),
|
||||||
|
lambda hf_id: "comfy-org/ideogram-4" in hf_id.lower(),
|
||||||
|
lambda hf_id: "comfy-org--ideogram-4" in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -58,3 +58,24 @@ class JoyEchoPipelineConfig(LTX2PipelineConfig):
|
|||||||
video_memory_frame_selection_mode: str = "center"
|
video_memory_frame_selection_mode: str = "center"
|
||||||
|
|
||||||
late_layer_ratio: float = 0.7
|
late_layer_ratio: float = 0.7
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.joy_echo import (
|
||||||
|
JoyEchoSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=JoyEchoSamplingParams,
|
||||||
|
pipeline_config_cls=JoyEchoPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"jdopensource/JoyAI-Echo",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
("joy-echo" in hf_id.lower() or "joyai-echo" in hf_id.lower())
|
||||||
|
and "image-edit" not in hf_id.lower()
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -428,3 +428,21 @@ class JoyImageEditPipelineConfig(ImagePipelineConfig):
|
|||||||
cond_norm = torch.norm(noise_pred_cond, dim=2, keepdim=True)
|
cond_norm = torch.norm(noise_pred_cond, dim=2, keepdim=True)
|
||||||
noise_norm = torch.norm(noise_pred, dim=2, keepdim=True).clamp_min(1e-12)
|
noise_norm = torch.norm(noise_pred, dim=2, keepdim=True).clamp_min(1e-12)
|
||||||
return noise_pred * (cond_norm / noise_norm)
|
return noise_pred * (cond_norm / noise_norm)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.joy_image import (
|
||||||
|
JoyImageEditSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=JoyImageEditSamplingParams,
|
||||||
|
pipeline_config_cls=JoyImageEditPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"jdopensource/JoyAI-Image-Edit-Diffusers",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "joyai-image-edit" in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -163,3 +163,17 @@ class Krea2PipelineConfig(ImagePipelineConfig):
|
|||||||
)
|
)
|
||||||
latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width)
|
latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width)
|
||||||
return latents
|
return latents
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.krea2 import (
|
||||||
|
Krea2SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Krea2SamplingParams,
|
||||||
|
pipeline_config_cls=Krea2PipelineConfig,
|
||||||
|
hf_model_paths=["krea/Krea-2"],
|
||||||
|
model_detectors=[lambda hf_id: "krea-2" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
|||||||
@@ -84,3 +84,18 @@ class LingBotVideoMoEPipelineConfig(PipelineConfig):
|
|||||||
1, -1, 1, 1, 1
|
1, -1, 1, 1, 1
|
||||||
)
|
)
|
||||||
return 1.0 / std, mean
|
return 1.0 / std, mean
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.lingbot_video_moe import (
|
||||||
|
LingBotVideoMoESamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LingBotVideoMoESamplingParams,
|
||||||
|
pipeline_config_cls=LingBotVideoMoEPipelineConfig,
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "lingbot-video-moe" in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -449,3 +449,26 @@ class LingBotWorldV2CausalDMDConfig(LingBotWorldCausalDMDConfig):
|
|||||||
dmd_denoising_steps: list[int] | None = field(
|
dmd_denoising_steps: list[int] | None = field(
|
||||||
default_factory=lambda: [1000, 750, 500, 250]
|
default_factory=lambda: [1000, 750, 500, 250]
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.lingbot_world import (
|
||||||
|
LingBotWorldSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LingBotWorldSamplingParams,
|
||||||
|
pipeline_config_cls=LingBotWorldCausalDMDConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"IPostYellow/lingbot-world-fast-diffusers",
|
||||||
|
"robbyant/lingbot-world-fast-diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LingBotWorldSamplingParams,
|
||||||
|
pipeline_config_cls=LingBotWorldV2CausalDMDConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"robbyant/lingbot-world-v2-14b-causal-fast-diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -604,3 +604,52 @@ class LongCatImageEditPipelineConfig(LongCatImagePipelineConfig):
|
|||||||
if latents.shape[1] > expected:
|
if latents.shape[1] > expected:
|
||||||
latents = latents[:, :expected, :]
|
latents = latents[:, :expected, :]
|
||||||
return super().post_denoising_loop(latents, batch)
|
return super().post_denoising_loop(latents, batch)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.longcat_image import (
|
||||||
|
LongCatImageEditSamplingParams,
|
||||||
|
LongCatImageEditTurboSamplingParams,
|
||||||
|
LongCatImageSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LongCatImageSamplingParams,
|
||||||
|
pipeline_config_cls=LongCatImagePipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"meituan-longcat/LongCat-Image",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "longcat" in hf_id.lower() and "edit" not in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
# LongCat-Image-Edit-Turbo (registered before Edit so its detector wins)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LongCatImageEditTurboSamplingParams,
|
||||||
|
pipeline_config_cls=LongCatImageEditPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"meituan-longcat/LongCat-Image-Edit-Turbo",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"longcat" in hf_id.lower()
|
||||||
|
and "edit" in hf_id.lower()
|
||||||
|
and "turbo" in hf_id.lower()
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LongCatImageEditSamplingParams,
|
||||||
|
pipeline_config_cls=LongCatImageEditPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"meituan-longcat/LongCat-Image-Edit",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"longcat" in hf_id.lower()
|
||||||
|
and "edit" in hf_id.lower()
|
||||||
|
and "turbo" not in hf_id.lower()
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -70,3 +70,19 @@ class LongLive2T2VConfig(Wan2_2_TI2V_5B_Config):
|
|||||||
super().__post_init__()
|
super().__post_init__()
|
||||||
self.vae_config.load_encoder = True
|
self.vae_config.load_encoder = True
|
||||||
self.vae_config.load_decoder = True
|
self.vae_config.load_decoder = True
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.longlive2 import (
|
||||||
|
LongLive2SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LongLive2SamplingParams,
|
||||||
|
pipeline_config_cls=LongLive2T2VConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Rabinovich/LongLive-2.0-5B-Diffusers",
|
||||||
|
"Efficient-Large-Model/LongLive-2.0-5B",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -726,3 +726,37 @@ class LTX23PipelineConfig(LTX2PipelineConfig):
|
|||||||
|
|
||||||
# original-mode lora swaps invalidate post-warmup timing calibration
|
# original-mode lora swaps invalidate post-warmup timing calibration
|
||||||
supports_auto_residency: bool = False
|
supports_auto_residency: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.ltx_2 import (
|
||||||
|
LTX2SamplingParams,
|
||||||
|
LTX23HQSamplingParams,
|
||||||
|
LTX23SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LTX2SamplingParams,
|
||||||
|
pipeline_config_cls=LTX2PipelineConfig,
|
||||||
|
hf_model_paths=["Lightricks/LTX-2"],
|
||||||
|
model_detectors=[
|
||||||
|
lambda path: "ltx" in path.lower() and "video" in path.lower(),
|
||||||
|
lambda path: (
|
||||||
|
"ltx-2" in path.lower()
|
||||||
|
and "ltx-2.3" not in path.lower()
|
||||||
|
and "ltx-2.5" not in path.lower()
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LTX23SamplingParams,
|
||||||
|
pipeline_config_cls=LTX23PipelineConfig,
|
||||||
|
hf_model_paths=["Lightricks/LTX-2.3"],
|
||||||
|
model_detectors=[
|
||||||
|
lambda path: "ltx-2.3" in path.lower(),
|
||||||
|
],
|
||||||
|
pipeline_config_registry_entries={
|
||||||
|
"LTX2TwoStageHQPipeline": (LTX2PipelineConfig, LTX23HQSamplingParams),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|||||||
@@ -55,3 +55,17 @@ class LTX25PipelineConfig(LTX2PipelineConfig):
|
|||||||
default_sigmas: tuple[float, ...] | None = field(
|
default_sigmas: tuple[float, ...] | None = field(
|
||||||
default_factory=lambda: LTX25_DISTILLED_SIGMA_VALUES
|
default_factory=lambda: LTX25_DISTILLED_SIGMA_VALUES
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.ltx_2_5 import LTX25SamplingParams
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=LTX25SamplingParams,
|
||||||
|
pipeline_config_cls=LTX25PipelineConfig,
|
||||||
|
hf_model_paths=["Lightricks/LTX-2.5-Diffusers"],
|
||||||
|
model_detectors=[
|
||||||
|
lambda path: "ltx-2.5" in path.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -356,3 +356,38 @@ class FastH3PipelineConfig(MiniMaxH3PipelineConfig):
|
|||||||
|
|
||||||
|
|
||||||
__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"]
|
__all__ = ["FastH3PipelineConfig", "MiniMaxH3PipelineConfig"]
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.minimax_h3 import (
|
||||||
|
FastH3SamplingParams,
|
||||||
|
MiniMaxH3SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=MiniMaxH3SamplingParams,
|
||||||
|
pipeline_config_cls=MiniMaxH3PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"MiniMaxAI/MiniMax-H3",
|
||||||
|
"MiniMax/MiniMax-H3",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda model_id: (
|
||||||
|
"minimaxh3" in model_id.lower().replace("-", "").replace("_", "")
|
||||||
|
and "vdn" not in model_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=FastH3SamplingParams,
|
||||||
|
pipeline_config_cls=FastH3PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda model_id: (
|
||||||
|
"fasth3" in model_id.lower().replace("-", "").replace("_", "")
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -74,3 +74,24 @@ class VDNH3PipelineConfig(MiniMaxH3PipelineConfig):
|
|||||||
|
|
||||||
|
|
||||||
__all__ = ["VDNH3PipelineConfig"]
|
__all__ = ["VDNH3PipelineConfig"]
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import (
|
||||||
|
VDNH3SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=VDNH3SamplingParams,
|
||||||
|
pipeline_config_cls=VDNH3PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"OpenVDN/vdn-minimax-h3",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda model_id: (
|
||||||
|
"vdn" in model_id.lower()
|
||||||
|
and "minimaxh3" in model_id.lower().replace("-", "").replace("_", "")
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -195,3 +195,26 @@ class MOVA720PConfig(MOVAPipelineConfig):
|
|||||||
"""Configuration for MOVA 720P (text+image -> video+audio) pipelines."""
|
"""Configuration for MOVA 720P (text+image -> video+audio) pipelines."""
|
||||||
|
|
||||||
max_area: int = 720 * 1280
|
max_area: int = 720 * 1280
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.mova import (
|
||||||
|
MOVA_360P_SamplingParams,
|
||||||
|
MOVA_720P_SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=MOVA_360P_SamplingParams,
|
||||||
|
pipeline_config_cls=MOVA360PConfig,
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "mova" in hf_id.lower() and "360p" in hf_id.lower()
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=MOVA_720P_SamplingParams,
|
||||||
|
pipeline_config_cls=MOVA720PConfig,
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "mova" in hf_id.lower() and "720p" in hf_id.lower()
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -164,3 +164,21 @@ class Pi05PipelineConfig(PipelineConfig):
|
|||||||
|
|
||||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||||
return ModelDeploymentConfig()
|
return ModelDeploymentConfig()
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.pi05 import Pi05SamplingParams
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Pi05SamplingParams,
|
||||||
|
pipeline_config_cls=Pi05PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"lerobot/pi05_base",
|
||||||
|
"lerobot/pi05_libero_base",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "pi05" in hf_id.lower(),
|
||||||
|
lambda hf_id: "pi0.5" in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -895,3 +895,72 @@ class QwenImageLayeredPipelineConfig(QwenImageEditPipelineConfig):
|
|||||||
latents = latents.permute(0, 2, 1, 3, 4).view(-1, c, 1, h, w)
|
latents = latents.permute(0, 2, 1, 3, 4).view(-1, c, 1, h, w)
|
||||||
# latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width)
|
# latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width)
|
||||||
return latents
|
return latents
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.qwenimage import (
|
||||||
|
QwenImage2512SamplingParams,
|
||||||
|
QwenImageEditPlusSamplingParams,
|
||||||
|
QwenImageLayeredSamplingParams,
|
||||||
|
QwenImageSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImageSamplingParams,
|
||||||
|
pipeline_config_cls=QwenImagePipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image", "nvidia/Qwen-Image-NVFP4"],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"qwen-image" in hf_id.lower()
|
||||||
|
and "edit" not in hf_id.lower()
|
||||||
|
and "layered" not in hf_id.lower()
|
||||||
|
and "2512" not in hf_id.lower()
|
||||||
|
and "qwen-image-2.1" not in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImage2512SamplingParams,
|
||||||
|
pipeline_config_cls=QwenImagePipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image-2512"],
|
||||||
|
model_detectors=[lambda hf_id: "qwen-image-2512" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImageSamplingParams,
|
||||||
|
pipeline_config_cls=QwenImageEditPipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image-Edit"],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"qwen-image-edit" in hf_id.lower()
|
||||||
|
and "2509" not in hf_id.lower()
|
||||||
|
and "2511" not in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
||||||
|
pipeline_config_cls=QwenImageEditPlusPipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image-Edit-2509"],
|
||||||
|
model_detectors=[lambda hf_id: "qwen-image-edit-2509" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
||||||
|
pipeline_config_cls=QwenImageEditPlus_2511_PipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image-Edit-2511"],
|
||||||
|
model_detectors=[lambda hf_id: "qwen-image-edit-2511" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImageLayeredSamplingParams,
|
||||||
|
pipeline_config_cls=QwenImageLayeredPipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image-Layered"],
|
||||||
|
model_detectors=[lambda hf_id: "qwen-image-layered" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
||||||
|
pipeline_config_cls=QwenImageEditPlusPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"FireRedTeam/FireRed-Image-Edit-1.0",
|
||||||
|
"FireRedTeam/FireRed-Image-Edit-1.1",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -86,3 +86,17 @@ class QwenImage21PipelineConfig(ImagePipelineConfig):
|
|||||||
|
|
||||||
def preprocess_condition_image(self, image, **kwargs):
|
def preprocess_condition_image(self, image, **kwargs):
|
||||||
return image
|
return image
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.qwenimage21 import (
|
||||||
|
QwenImage21SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=QwenImage21SamplingParams,
|
||||||
|
pipeline_config_cls=QwenImage21PipelineConfig,
|
||||||
|
hf_model_paths=["Qwen/Qwen-Image-2.1"],
|
||||||
|
model_detectors=[lambda hf_id: "qwen-image-2.1" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
|||||||
@@ -127,3 +127,30 @@ class SanaPipelineConfig(SpatialImagePipelineConfig):
|
|||||||
|
|
||||||
def gather_latents_for_sp(self, latents):
|
def gather_latents_for_sp(self, latents):
|
||||||
return latents
|
return latents
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.sana import SanaSamplingParams
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=SanaSamplingParams,
|
||||||
|
pipeline_config_cls=SanaPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers",
|
||||||
|
"Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers",
|
||||||
|
"Efficient-Large-Model/Sana_1600M_1024px_diffusers",
|
||||||
|
"Efficient-Large-Model/Sana_600M_1024px_diffusers",
|
||||||
|
"Efficient-Large-Model/Sana_1600M_512px_diffusers",
|
||||||
|
"Efficient-Large-Model/Sana_600M_512px_diffusers",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"sana" in hf_id.lower()
|
||||||
|
and "sana-wm" not in hf_id.lower()
|
||||||
|
and "sana_wm" not in hf_id.lower()
|
||||||
|
and "sana-video" not in hf_id.lower()
|
||||||
|
and "sana_video" not in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -116,3 +116,19 @@ class SanaVideoPipelineConfig(PipelineConfig):
|
|||||||
|
|
||||||
def gather_latents_for_sp(self, latents, batch=None):
|
def gather_latents_for_sp(self, latents, batch=None):
|
||||||
return latents
|
return latents
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=SanaVideoSamplingParams,
|
||||||
|
pipeline_config_cls=SanaVideoPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "sana-video" in hf_id.lower() or "sana_video" in hf_id.lower()
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -339,3 +339,20 @@ class SanaWMRealtimeConfig(SanaWMPipelineConfig):
|
|||||||
keep_resident_components=("dit",),
|
keep_resident_components=("dit",),
|
||||||
auto_enable_cfg_parallel=False,
|
auto_enable_cfg_parallel=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.sana_wm import SanaWMSamplingParams
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=SanaWMSamplingParams,
|
||||||
|
pipeline_config_cls=SanaWMPipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Efficient-Large-Model/SANA-WM_bidirectional",
|
||||||
|
"Efficient-Large-Model/SANA-WM_streaming",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "sana-wm" in hf_id.lower() or "sana_wm" in hf_id.lower(),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -197,3 +197,18 @@ class SenseNovaU1PipelineConfig(PipelineConfig):
|
|||||||
auto_enable_cfg_parallel=False,
|
auto_enable_cfg_parallel=False,
|
||||||
supports_cfg_parallel=False,
|
supports_cfg_parallel=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.sensenova_u1 import (
|
||||||
|
SenseNovaU1SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=SenseNovaU1SamplingParams,
|
||||||
|
pipeline_config_cls=SenseNovaU1PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"sensenova/SenseNova-U1.5-8B-MoT",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -200,3 +200,33 @@ class StableDiffusion3PipelineConfig(SpatialImagePipelineConfig):
|
|||||||
batch.height // spatial_ratio,
|
batch.height // spatial_ratio,
|
||||||
batch.width // spatial_ratio,
|
batch.width // spatial_ratio,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.stablediffusion3 import (
|
||||||
|
StableDiffusion3SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=StableDiffusion3SamplingParams,
|
||||||
|
pipeline_config_cls=StableDiffusion3PipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"stabilityai/stable-diffusion-3-medium",
|
||||||
|
"stabilityai/stable-diffusion-3-medium-diffusers",
|
||||||
|
"stabilityai/stable-diffusion-3.5-medium",
|
||||||
|
"stabilityai/stable-diffusion-3.5-medium-diffusers",
|
||||||
|
"stabilityai/stable-diffusion-3.5-large",
|
||||||
|
"stabilityai/stable-diffusion-3.5-large-diffusers",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: (
|
||||||
|
"stable-diffusion-3-medium" in hf_id.lower()
|
||||||
|
or "stable-diffusion-3.5-medium" in hf_id.lower()
|
||||||
|
or "stable-diffusion-3.5-large" in hf_id.lower()
|
||||||
|
or "sd3-medium" in hf_id.lower()
|
||||||
|
or "sd3.5-medium" in hf_id.lower()
|
||||||
|
or "sd3.5-large" in hf_id.lower()
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -312,3 +312,114 @@ class SelfForcingWanT2V480PConfig(WanT2V480PConfig):
|
|||||||
default_factory=lambda: [1000, 750, 500, 250]
|
default_factory=lambda: [1000, 750, 500, 250]
|
||||||
)
|
)
|
||||||
warp_denoising_step: bool = True
|
warp_denoising_step: bool = True
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.wan import (
|
||||||
|
FastWanT2V480PConfig,
|
||||||
|
Turbo_Wan2_2_I2V_A14B_SamplingParam,
|
||||||
|
Wan2_1_Fun_1_3B_InP_SamplingParams,
|
||||||
|
Wan2_2_I2V_A14B_SamplingParam,
|
||||||
|
Wan2_2_T2V_A14B_SamplingParam,
|
||||||
|
Wan2_2_TI2V_5B_SamplingParam,
|
||||||
|
WanI2V_14B_480P_SamplingParam,
|
||||||
|
WanI2V_14B_720P_SamplingParam,
|
||||||
|
WanT2V_1_3B_SamplingParams,
|
||||||
|
WanT2V_14B_SamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=WanT2V_1_3B_SamplingParams,
|
||||||
|
pipeline_config_cls=WanT2V480PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "wanpipeline" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=WanT2V_1_3B_SamplingParams,
|
||||||
|
pipeline_config_cls=TurboWanT2V1_3B480PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=WanT2V_14B_SamplingParams,
|
||||||
|
pipeline_config_cls=WanT2V720PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=WanT2V_14B_SamplingParams,
|
||||||
|
pipeline_config_cls=TurboWanT2V480PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"IPostYellow/TurboWan2.1-T2V-14B-Diffusers",
|
||||||
|
"IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=WanI2V_14B_480P_SamplingParam,
|
||||||
|
pipeline_config_cls=WanI2V480PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "wanimagetovideo" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=WanI2V_14B_720P_SamplingParam,
|
||||||
|
pipeline_config_cls=WanI2V720PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Turbo_Wan2_2_I2V_A14B_SamplingParam,
|
||||||
|
pipeline_config_cls=TurboWanI2V720Config,
|
||||||
|
hf_model_paths=[
|
||||||
|
"IPostYellow/TurboWan2.2-I2V-A14B-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParams,
|
||||||
|
pipeline_config_cls=WanI2V480PConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam,
|
||||||
|
pipeline_config_cls=Wan2_2_TI2V_5B_Config,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam,
|
||||||
|
pipeline_config_cls=FastWan2_2_TI2V_5B_Config,
|
||||||
|
hf_model_paths=[
|
||||||
|
"FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
||||||
|
"FastVideo/FastWan2.2-TI2V-5B-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Wan2_2_T2V_A14B_SamplingParam,
|
||||||
|
pipeline_config_cls=Wan2_2_T2V_A14B_Config,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||||
|
"nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=Wan2_2_I2V_A14B_SamplingParam,
|
||||||
|
pipeline_config_cls=Wan2_2_I2V_A14B_Config,
|
||||||
|
hf_model_paths=["Wan-AI/Wan2.2-I2V-A14B-Diffusers"],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=FastWanT2V480PConfig,
|
||||||
|
pipeline_config_cls=FastWan2_1_T2V_480P_Config,
|
||||||
|
hf_model_paths=[
|
||||||
|
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -515,3 +515,30 @@ class ZImagePipelineConfig(ZImageRolloutPipelineMixin, ImagePipelineConfig):
|
|||||||
dtype=torch.long,
|
dtype=torch.long,
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def register():
|
||||||
|
from sglang.multimodal_gen.configs.sample.zimage import (
|
||||||
|
ZImageSamplingParams,
|
||||||
|
ZImageTurboSamplingParams,
|
||||||
|
)
|
||||||
|
from sglang.multimodal_gen.registry import register_configs
|
||||||
|
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=ZImageTurboSamplingParams,
|
||||||
|
pipeline_config_cls=ZImagePipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Tongyi-MAI/Z-Image-Turbo",
|
||||||
|
],
|
||||||
|
model_detectors=[lambda hf_id: "z-image-turbo" in hf_id.lower()],
|
||||||
|
)
|
||||||
|
register_configs(
|
||||||
|
sampling_param_cls=ZImageSamplingParams,
|
||||||
|
pipeline_config_cls=ZImagePipelineConfig,
|
||||||
|
hf_model_paths=[
|
||||||
|
"Tongyi-MAI/Z-Image",
|
||||||
|
],
|
||||||
|
model_detectors=[
|
||||||
|
lambda hf_id: "z-image" in hf_id.lower() and "turbo" not in hf_id.lower()
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|||||||
@@ -27,193 +27,7 @@ from typing import (
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from sglang.multimodal_gen.runtime.server_args import Backend
|
from sglang.multimodal_gen.runtime.server_args import Backend
|
||||||
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs import (
|
|
||||||
Cosmos3Config,
|
|
||||||
FastH3PipelineConfig,
|
|
||||||
FastHunyuanConfig,
|
|
||||||
FluxPipelineConfig,
|
|
||||||
HeliosDistilledConfig,
|
|
||||||
HeliosMidConfig,
|
|
||||||
HeliosT2VConfig,
|
|
||||||
HunyuanConfig,
|
|
||||||
LingBotWorldCausalDMDConfig,
|
|
||||||
LingBotWorldV2CausalDMDConfig,
|
|
||||||
MiniMaxH3PipelineConfig,
|
|
||||||
WanI2V480PConfig,
|
|
||||||
WanI2V720PConfig,
|
|
||||||
WanT2V480PConfig,
|
|
||||||
WanT2V720PConfig,
|
|
||||||
ZImagePipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.ernie_image import (
|
|
||||||
ErnieImagePipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.flux import (
|
|
||||||
Flux2KleinBasePipelineConfig,
|
|
||||||
Flux2KleinPipelineConfig,
|
|
||||||
Flux2PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.glm_image import (
|
|
||||||
GlmImagePipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan3d import (
|
|
||||||
Hunyuan3D2PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.ideogram import (
|
|
||||||
Ideogram4DistilledPipelineConfig,
|
|
||||||
Ideogram4PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.joy_echo import (
|
|
||||||
JoyEchoPipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.joy_image import (
|
|
||||||
JoyImageEditPipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.krea2 import Krea2PipelineConfig
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.lingbot_video_moe import (
|
|
||||||
LingBotVideoMoEPipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import (
|
|
||||||
LongCatImageEditPipelineConfig,
|
|
||||||
LongCatImagePipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.longlive2 import LongLive2T2VConfig
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import (
|
|
||||||
LTX2PipelineConfig,
|
|
||||||
LTX23PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2_5 import LTX25PipelineConfig
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.minimax_h3_vdn import (
|
|
||||||
VDNH3PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.mova import (
|
|
||||||
MOVA360PConfig,
|
|
||||||
MOVA720PConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.pi05 import Pi05PipelineConfig
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
|
||||||
QwenImageEditPipelineConfig,
|
|
||||||
QwenImageEditPlus_2511_PipelineConfig,
|
|
||||||
QwenImageEditPlusPipelineConfig,
|
|
||||||
QwenImageLayeredPipelineConfig,
|
|
||||||
QwenImagePipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image21 import (
|
|
||||||
QwenImage21PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.sana import SanaPipelineConfig
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.sana_video import (
|
|
||||||
SanaVideoPipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.sana_wm import SanaWMPipelineConfig
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.sensenova_u1 import (
|
|
||||||
SenseNovaU1PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.stablediffusion3 import (
|
|
||||||
StableDiffusion3PipelineConfig,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
|
||||||
FastWan2_1_T2V_480P_Config,
|
|
||||||
FastWan2_2_TI2V_5B_Config,
|
|
||||||
TurboWanI2V720Config,
|
|
||||||
TurboWanT2V1_3B480PConfig,
|
|
||||||
TurboWanT2V480PConfig,
|
|
||||||
Wan2_2_I2V_A14B_Config,
|
|
||||||
Wan2_2_T2V_A14B_Config,
|
|
||||||
Wan2_2_TI2V_5B_Config,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.ernie_image import ErnieImageSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.flux import (
|
|
||||||
Flux2KleinBaseSamplingParams,
|
|
||||||
Flux2KleinSamplingParams,
|
|
||||||
Flux2SamplingParams,
|
|
||||||
FluxSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.glmimage import GlmImageSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.helios import (
|
|
||||||
HeliosDistilledSamplingParams,
|
|
||||||
HeliosMidSamplingParams,
|
|
||||||
HeliosT2VSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.hunyuan import (
|
|
||||||
FastHunyuanSamplingParam,
|
|
||||||
HunyuanSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.hunyuan3d import Hunyuan3DSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.ideogram import (
|
|
||||||
Ideogram4FastSamplingParams,
|
|
||||||
Ideogram4InstantSamplingParams,
|
|
||||||
Ideogram4SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.joy_echo import JoyEchoSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.joy_image import (
|
|
||||||
JoyImageEditSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.krea2 import (
|
|
||||||
Krea2SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.lingbot_video_moe import (
|
|
||||||
LingBotVideoMoESamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.lingbot_world import (
|
|
||||||
LingBotWorldSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.longcat_image import (
|
|
||||||
LongCatImageEditSamplingParams,
|
|
||||||
LongCatImageEditTurboSamplingParams,
|
|
||||||
LongCatImageSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.longlive2 import LongLive2SamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.ltx_2 import (
|
|
||||||
LTX2SamplingParams,
|
|
||||||
LTX23HQSamplingParams,
|
|
||||||
LTX23SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.ltx_2_5 import LTX25SamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import (
|
|
||||||
FastH3SamplingParams,
|
|
||||||
MiniMaxH3SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import VDNH3SamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.mova import (
|
|
||||||
MOVA_360P_SamplingParams,
|
|
||||||
MOVA_720P_SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.pi05 import Pi05SamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.qwenimage import (
|
|
||||||
QwenImage2512SamplingParams,
|
|
||||||
QwenImageEditPlusSamplingParams,
|
|
||||||
QwenImageLayeredSamplingParams,
|
|
||||||
QwenImageSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.qwenimage21 import QwenImage21SamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.sana import SanaSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.sana_wm import SanaWMSamplingParams
|
|
||||||
from sglang.multimodal_gen.configs.sample.sensenova_u1 import (
|
|
||||||
SenseNovaU1SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.stablediffusion3 import (
|
|
||||||
StableDiffusion3SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.wan import (
|
|
||||||
FastWanT2V480PConfig,
|
|
||||||
Turbo_Wan2_2_I2V_A14B_SamplingParam,
|
|
||||||
Wan2_1_Fun_1_3B_InP_SamplingParams,
|
|
||||||
Wan2_2_I2V_A14B_SamplingParam,
|
|
||||||
Wan2_2_T2V_A14B_SamplingParam,
|
|
||||||
Wan2_2_TI2V_5B_SamplingParam,
|
|
||||||
WanI2V_14B_480P_SamplingParam,
|
|
||||||
WanI2V_14B_720P_SamplingParam,
|
|
||||||
WanT2V_1_3B_SamplingParams,
|
|
||||||
WanT2V_14B_SamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sample.zimage import (
|
|
||||||
ZImageSamplingParams,
|
|
||||||
ZImageTurboSamplingParams,
|
|
||||||
)
|
|
||||||
from sglang.multimodal_gen.configs.sensenova_u1 import (
|
from sglang.multimodal_gen.configs.sensenova_u1 import (
|
||||||
SENSENOVA_U1_MODEL_IDS,
|
SENSENOVA_U1_MODEL_IDS,
|
||||||
is_sensenova_u1_adapter_only_model,
|
is_sensenova_u1_adapter_only_model,
|
||||||
@@ -375,6 +189,9 @@ def register_configs(
|
|||||||
pipeline_config_cls: Type[PipelineConfig],
|
pipeline_config_cls: Type[PipelineConfig],
|
||||||
hf_model_paths: Optional[List[str]] = None,
|
hf_model_paths: Optional[List[str]] = None,
|
||||||
model_detectors: Optional[List[Callable[[str], bool]]] = None,
|
model_detectors: Optional[List[Callable[[str], bool]]] = None,
|
||||||
|
pipeline_config_registry_entries: Optional[
|
||||||
|
Dict[str, Tuple[Type[PipelineConfig], Type[Any]]]
|
||||||
|
] = None,
|
||||||
) -> str:
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Registers configuration classes for a new model family.
|
Registers configuration classes for a new model family.
|
||||||
@@ -396,6 +213,11 @@ def register_configs(
|
|||||||
if model_detectors:
|
if model_detectors:
|
||||||
for detector in model_detectors:
|
for detector in model_detectors:
|
||||||
_MODEL_NAME_DETECTORS.append((model_id, detector))
|
_MODEL_NAME_DETECTORS.append((model_id, detector))
|
||||||
|
|
||||||
|
if pipeline_config_registry_entries:
|
||||||
|
for pipeline_name, (pc_cls, sp_cls) in pipeline_config_registry_entries.items():
|
||||||
|
_PIPELINE_CONFIG_REGISTRY.setdefault(pipeline_name, (pc_cls, sp_cls))
|
||||||
|
|
||||||
return model_id
|
return model_id
|
||||||
|
|
||||||
|
|
||||||
@@ -459,6 +281,56 @@ def register_pipeline(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_configs_discovered: bool = False
|
||||||
|
|
||||||
|
# SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match)
|
||||||
|
# SANA-Video (register before generic SANA to avoid detector overlap).
|
||||||
|
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video")
|
||||||
|
|
||||||
|
|
||||||
|
def _discover_and_register_configs() -> None:
|
||||||
|
global _configs_discovered
|
||||||
|
if _configs_discovered:
|
||||||
|
return
|
||||||
|
_configs_discovered = True
|
||||||
|
|
||||||
|
package_name = "sglang.multimodal_gen.configs.pipeline_configs"
|
||||||
|
package = importlib.import_module(package_name)
|
||||||
|
|
||||||
|
discovered = []
|
||||||
|
for _, module_name, ispkg in pkgutil.walk_packages(
|
||||||
|
package.__path__, package.__name__ + "."
|
||||||
|
):
|
||||||
|
if not ispkg:
|
||||||
|
try:
|
||||||
|
config_module = importlib.import_module(module_name)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
f"Skipping config module {module_name} during discovery due to import failure: {exc}",
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if hasattr(config_module, "register"):
|
||||||
|
discovered.append((module_name, config_module))
|
||||||
|
|
||||||
|
def _sort_key(item):
|
||||||
|
short_name = item[0].rsplit(".", 1)[-1]
|
||||||
|
try:
|
||||||
|
return (0, _CONFIG_REGISTER_PRIORITY.index(short_name))
|
||||||
|
except ValueError:
|
||||||
|
return (1, 0)
|
||||||
|
|
||||||
|
discovered.sort(key=_sort_key)
|
||||||
|
|
||||||
|
for module_name, config_module in discovered:
|
||||||
|
try:
|
||||||
|
config_module.register()
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
f"register() failed for {module_name}: {exc}",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_model_short_name(model_id: str) -> str:
|
def get_model_short_name(model_id: str) -> str:
|
||||||
if "/" in model_id:
|
if "/" in model_id:
|
||||||
return model_id.rstrip("/").split("/")[-1]
|
return model_id.rstrip("/").split("/")[-1]
|
||||||
@@ -809,657 +681,7 @@ def get_model_info(
|
|||||||
return model_info
|
return model_info
|
||||||
|
|
||||||
|
|
||||||
# Registration of model configs
|
_discover_and_register_configs()
|
||||||
def _register_configs():
|
|
||||||
# Pi0.5 / OpenPI / LeRobot action policies.
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Pi05SamplingParams,
|
|
||||||
pipeline_config_cls=Pi05PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"lerobot/pi05_base",
|
|
||||||
"lerobot/pi05_libero_base",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "pi05" in hf_id.lower(),
|
|
||||||
lambda hf_id: "pi0.5" in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# LTX-2
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LTX2SamplingParams,
|
|
||||||
pipeline_config_cls=LTX2PipelineConfig,
|
|
||||||
hf_model_paths=["Lightricks/LTX-2"],
|
|
||||||
model_detectors=[
|
|
||||||
lambda path: "ltx" in path.lower() and "video" in path.lower(),
|
|
||||||
lambda path: (
|
|
||||||
"ltx-2" in path.lower()
|
|
||||||
and "ltx-2.3" not in path.lower()
|
|
||||||
and "ltx-2.5" not in path.lower()
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LTX23SamplingParams,
|
|
||||||
pipeline_config_cls=LTX23PipelineConfig,
|
|
||||||
hf_model_paths=["Lightricks/LTX-2.3"],
|
|
||||||
model_detectors=[
|
|
||||||
lambda path: "ltx-2.3" in path.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# Keeps the LTX-2 pipeline class; only component geometry and the pinned
|
|
||||||
# distilled schedule differ. Only the `-Diffusers` repo is listed --
|
|
||||||
# `Lightricks/LTX-2.5` is a split pack of bare `.safetensors` and would need
|
|
||||||
# a model overlay first.
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LTX25SamplingParams,
|
|
||||||
pipeline_config_cls=LTX25PipelineConfig,
|
|
||||||
hf_model_paths=["Lightricks/LTX-2.5-Diffusers"],
|
|
||||||
model_detectors=[
|
|
||||||
lambda path: "ltx-2.5" in path.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# register dedicated sampling params for LTX2TwoStageHQPipeline
|
|
||||||
_PIPELINE_CONFIG_REGISTRY.setdefault(
|
|
||||||
"LTX2TwoStageHQPipeline",
|
|
||||||
(LTX2PipelineConfig, LTX23HQSamplingParams),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Hunyuan
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=HunyuanSamplingParams,
|
|
||||||
pipeline_config_cls=HunyuanConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"hunyuanvideo-community/HunyuanVideo",
|
|
||||||
],
|
|
||||||
model_detectors=[lambda hf_id: "hunyuanvideo" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=FastHunyuanSamplingParam,
|
|
||||||
pipeline_config_cls=FastHunyuanConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"FastVideo/FastHunyuan-diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# Wan
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=WanT2V_1_3B_SamplingParams,
|
|
||||||
pipeline_config_cls=WanT2V480PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
|
||||||
],
|
|
||||||
model_detectors=[lambda hf_id: "wanpipeline" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=WanT2V_1_3B_SamplingParams,
|
|
||||||
pipeline_config_cls=TurboWanT2V1_3B480PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"IPostYellow/TurboWan2.1-T2V-1.3B-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=WanT2V_14B_SamplingParams,
|
|
||||||
pipeline_config_cls=WanT2V720PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Wan-AI/Wan2.1-T2V-14B-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=WanT2V_14B_SamplingParams,
|
|
||||||
pipeline_config_cls=TurboWanT2V480PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"IPostYellow/TurboWan2.1-T2V-14B-Diffusers",
|
|
||||||
"IPostYellow/TurboWan2.1-T2V-14B-720P-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=WanI2V_14B_480P_SamplingParam,
|
|
||||||
pipeline_config_cls=WanI2V480PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
|
||||||
],
|
|
||||||
model_detectors=[lambda hf_id: "wanimagetovideo" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=WanI2V_14B_720P_SamplingParam,
|
|
||||||
pipeline_config_cls=WanI2V720PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Wan-AI/Wan2.1-I2V-14B-720P-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Turbo_Wan2_2_I2V_A14B_SamplingParam,
|
|
||||||
pipeline_config_cls=TurboWanI2V720Config,
|
|
||||||
hf_model_paths=[
|
|
||||||
"IPostYellow/TurboWan2.2-I2V-A14B-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Wan2_1_Fun_1_3B_InP_SamplingParams,
|
|
||||||
pipeline_config_cls=WanI2V480PConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"weizhou03/Wan2.1-Fun-1.3B-InP-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam,
|
|
||||||
pipeline_config_cls=Wan2_2_TI2V_5B_Config,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Wan2_2_TI2V_5B_SamplingParam,
|
|
||||||
pipeline_config_cls=FastWan2_2_TI2V_5B_Config,
|
|
||||||
hf_model_paths=[
|
|
||||||
"FastVideo/FastWan2.2-TI2V-5B-FullAttn-Diffusers",
|
|
||||||
"FastVideo/FastWan2.2-TI2V-5B-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Wan2_2_T2V_A14B_SamplingParam,
|
|
||||||
pipeline_config_cls=Wan2_2_T2V_A14B_Config,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
|
||||||
"nvidia/Wan2.2-T2V-A14B-Diffusers-NVFP4",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Wan2_2_I2V_A14B_SamplingParam,
|
|
||||||
pipeline_config_cls=Wan2_2_I2V_A14B_Config,
|
|
||||||
hf_model_paths=["Wan-AI/Wan2.2-I2V-A14B-Diffusers"],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LingBotWorldSamplingParams,
|
|
||||||
pipeline_config_cls=LingBotWorldCausalDMDConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"IPostYellow/lingbot-world-fast-diffusers",
|
|
||||||
"robbyant/lingbot-world-fast-diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LingBotWorldSamplingParams,
|
|
||||||
pipeline_config_cls=LingBotWorldV2CausalDMDConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"robbyant/lingbot-world-v2-14b-causal-fast-diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LongLive2SamplingParams,
|
|
||||||
pipeline_config_cls=LongLive2T2VConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
# Since LongLive-2.0-5B does not have official diffusers release
|
|
||||||
"Rabinovich/LongLive-2.0-5B-Diffusers",
|
|
||||||
"Efficient-Large-Model/LongLive-2.0-5B",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=FastWanT2V480PConfig,
|
|
||||||
pipeline_config_cls=FastWan2_1_T2V_480P_Config,
|
|
||||||
hf_model_paths=[
|
|
||||||
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# MOVA
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=MOVA_360P_SamplingParams,
|
|
||||||
pipeline_config_cls=MOVA360PConfig,
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "mova" in hf_id.lower() and "360p" in hf_id.lower()
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=MOVA_720P_SamplingParams,
|
|
||||||
pipeline_config_cls=MOVA720PConfig,
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "mova" in hf_id.lower() and "720p" in hf_id.lower()
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=MiniMaxH3SamplingParams,
|
|
||||||
pipeline_config_cls=MiniMaxH3PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"MiniMaxAI/MiniMax-H3",
|
|
||||||
"MiniMax/MiniMax-H3",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda model_id: (
|
|
||||||
"minimaxh3" in model_id.lower().replace("-", "").replace("_", "")
|
|
||||||
and "vdn" not in model_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=SenseNovaU1SamplingParams,
|
|
||||||
pipeline_config_cls=SenseNovaU1PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"sensenova/SenseNova-U1.5-8B-MoT",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=FastH3SamplingParams,
|
|
||||||
pipeline_config_cls=FastH3PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"FastVideo/FastVideo-FastH3-4-step-Preview-v1-VSA-DataFree",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda model_id: (
|
|
||||||
"fasth3" in model_id.lower().replace("-", "").replace("_", "")
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=VDNH3SamplingParams,
|
|
||||||
pipeline_config_cls=VDNH3PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"OpenVDN/vdn-minimax-h3",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda model_id: (
|
|
||||||
"vdn" in model_id.lower()
|
|
||||||
and "minimaxh3" in model_id.lower().replace("-", "").replace("_", "")
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# FLUX
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=FluxSamplingParams,
|
|
||||||
pipeline_config_cls=FluxPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"black-forest-labs/FLUX.1-dev",
|
|
||||||
],
|
|
||||||
model_detectors=[lambda hf_id: "flux.1" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Flux2KleinSamplingParams,
|
|
||||||
pipeline_config_cls=Flux2KleinPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"black-forest-labs/FLUX.2-klein-4B",
|
|
||||||
"black-forest-labs/FLUX.2-klein-9B",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower())
|
|
||||||
and "base" not in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Flux2KleinBaseSamplingParams,
|
|
||||||
pipeline_config_cls=Flux2KleinBasePipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"black-forest-labs/FLUX.2-klein-base-4B",
|
|
||||||
"black-forest-labs/FLUX.2-klein-base-9B",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
("flux.2-klein" in hf_id.lower() or "flux2-klein" in hf_id.lower())
|
|
||||||
and "base" in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Flux2SamplingParams,
|
|
||||||
pipeline_config_cls=Flux2PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"black-forest-labs/FLUX.2-dev",
|
|
||||||
"black-forest-labs/FLUX.2-dev-NVFP4",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "flux.2" in hf_id.lower() and "klein" not in hf_id.lower()
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=ZImageTurboSamplingParams,
|
|
||||||
pipeline_config_cls=ZImagePipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Tongyi-MAI/Z-Image-Turbo",
|
|
||||||
],
|
|
||||||
model_detectors=[lambda hf_id: "z-image-turbo" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=ZImageSamplingParams,
|
|
||||||
pipeline_config_cls=ZImagePipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Tongyi-MAI/Z-Image",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "z-image" in hf_id.lower() and "turbo" not in hf_id.lower()
|
|
||||||
],
|
|
||||||
)
|
|
||||||
# Krea-2 (K2)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Krea2SamplingParams,
|
|
||||||
pipeline_config_cls=Krea2PipelineConfig,
|
|
||||||
hf_model_paths=["krea/Krea-2"],
|
|
||||||
model_detectors=[lambda hf_id: "krea-2" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
# Qwen-Image
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImage21SamplingParams,
|
|
||||||
pipeline_config_cls=QwenImage21PipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image-2.1"],
|
|
||||||
model_detectors=[lambda hf_id: "qwen-image-2.1" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImageSamplingParams,
|
|
||||||
pipeline_config_cls=QwenImagePipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image", "nvidia/Qwen-Image-NVFP4"],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"qwen-image" in hf_id.lower()
|
|
||||||
and "edit" not in hf_id.lower()
|
|
||||||
and "layered" not in hf_id.lower()
|
|
||||||
and "2512" not in hf_id.lower()
|
|
||||||
and "qwen-image-2.1" not in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImage2512SamplingParams,
|
|
||||||
pipeline_config_cls=QwenImagePipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image-2512"],
|
|
||||||
model_detectors=[lambda hf_id: "qwen-image-2512" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImageSamplingParams,
|
|
||||||
pipeline_config_cls=QwenImageEditPipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image-Edit"],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"qwen-image-edit" in hf_id.lower()
|
|
||||||
and "2509" not in hf_id.lower()
|
|
||||||
and "2511" not in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
|
||||||
pipeline_config_cls=QwenImageEditPlusPipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image-Edit-2509"],
|
|
||||||
model_detectors=[lambda hf_id: "qwen-image-edit-2509" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
|
||||||
pipeline_config_cls=QwenImageEditPlus_2511_PipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image-Edit-2511"],
|
|
||||||
model_detectors=[lambda hf_id: "qwen-image-edit-2511" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImageLayeredSamplingParams,
|
|
||||||
pipeline_config_cls=QwenImageLayeredPipelineConfig,
|
|
||||||
hf_model_paths=["Qwen/Qwen-Image-Layered"],
|
|
||||||
model_detectors=[lambda hf_id: "qwen-image-layered" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=StableDiffusion3SamplingParams,
|
|
||||||
pipeline_config_cls=StableDiffusion3PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"stabilityai/stable-diffusion-3-medium",
|
|
||||||
"stabilityai/stable-diffusion-3-medium-diffusers",
|
|
||||||
"stabilityai/stable-diffusion-3.5-medium",
|
|
||||||
"stabilityai/stable-diffusion-3.5-medium-diffusers",
|
|
||||||
"stabilityai/stable-diffusion-3.5-large",
|
|
||||||
"stabilityai/stable-diffusion-3.5-large-diffusers",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"stable-diffusion-3-medium" in hf_id.lower()
|
|
||||||
or "stable-diffusion-3.5-medium" in hf_id.lower()
|
|
||||||
or "stable-diffusion-3.5-large" in hf_id.lower()
|
|
||||||
or "sd3-medium" in hf_id.lower()
|
|
||||||
or "sd3.5-medium" in hf_id.lower()
|
|
||||||
or "sd3.5-large" in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=GlmImageSamplingParams,
|
|
||||||
pipeline_config_cls=GlmImagePipelineConfig,
|
|
||||||
model_detectors=[lambda hf_id: "glm-image" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Hunyuan3DSamplingParams,
|
|
||||||
pipeline_config_cls=Hunyuan3D2PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"tencent/Hunyuan3D-2",
|
|
||||||
],
|
|
||||||
model_detectors=[lambda hf_id: "hunyuan3d" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Helios
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=HeliosT2VSamplingParams,
|
|
||||||
pipeline_config_cls=HeliosT2VConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"BestWishYsh/Helios-Base",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"helios" in hf_id.lower()
|
|
||||||
and "mid" not in hf_id.lower()
|
|
||||||
and "distill" not in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=HeliosMidSamplingParams,
|
|
||||||
pipeline_config_cls=HeliosMidConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"BestWishYsh/Helios-Mid",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=HeliosDistilledSamplingParams,
|
|
||||||
pipeline_config_cls=HeliosDistilledConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"BestWishYsh/Helios-Distilled",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=SanaWMSamplingParams,
|
|
||||||
pipeline_config_cls=SanaWMPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Efficient-Large-Model/SANA-WM_bidirectional",
|
|
||||||
"Efficient-Large-Model/SANA-WM_streaming",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
# Match "sana-wm" or "sana_wm" but NOT plain T2I "sana" checkpoints.
|
|
||||||
lambda hf_id: "sana-wm" in hf_id.lower() or "sana_wm" in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# SANA-Video (register before generic SANA to avoid detector overlap).
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=SanaVideoSamplingParams,
|
|
||||||
pipeline_config_cls=SanaVideoPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Efficient-Large-Model/SANA-Video_2B_480p_diffusers",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "sana-video" in hf_id.lower() or "sana_video" in hf_id.lower()
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Cosmos3 — single checkpoint serves T2V, I2V, and T2I. Mode is dispatched
|
|
||||||
# per-request inside the pipeline from ``num_frames`` and ``image_path``.
|
|
||||||
# All variants share the same pipeline; arch dimensions (size, activation,
|
|
||||||
# QK-norm) come from ``transformer/config.json`` via ``update_model_arch``.
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Cosmos3SamplingParams,
|
|
||||||
pipeline_config_cls=Cosmos3Config,
|
|
||||||
hf_model_paths=[
|
|
||||||
"nvidia/Cosmos3-Nano",
|
|
||||||
"nvidia/Cosmos3-Nano-Policy-DROID",
|
|
||||||
"nvidia/Cosmos3-Super",
|
|
||||||
"nvidia/Cosmos3-Super-Text2Image",
|
|
||||||
"nvidia/Cosmos3-Super-Image2Video",
|
|
||||||
"nvidia/Cosmos3-Edge",
|
|
||||||
],
|
|
||||||
# Match both the new ``Cosmos3OmniPipeline`` and the legacy
|
|
||||||
# ``Cosmos3OmniDiffusersPipeline`` ``_class_name`` (diffusers rename).
|
|
||||||
model_detectors=[lambda hf_id: "cosmos3omni" in hf_id.lower()],
|
|
||||||
)
|
|
||||||
|
|
||||||
# SANA
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=SanaSamplingParams,
|
|
||||||
pipeline_config_cls=SanaPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"Efficient-Large-Model/SANA1.5_1.6B_1024px_diffusers",
|
|
||||||
"Efficient-Large-Model/SANA1.5_4.8B_1024px_diffusers",
|
|
||||||
"Efficient-Large-Model/Sana_1600M_1024px_diffusers",
|
|
||||||
"Efficient-Large-Model/Sana_600M_1024px_diffusers",
|
|
||||||
"Efficient-Large-Model/Sana_1600M_512px_diffusers",
|
|
||||||
"Efficient-Large-Model/Sana_600M_512px_diffusers",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"sana" in hf_id.lower()
|
|
||||||
and "sana-wm" not in hf_id.lower()
|
|
||||||
and "sana_wm" not in hf_id.lower()
|
|
||||||
and "sana-video" not in hf_id.lower()
|
|
||||||
and "sana_video" not in hf_id.lower()
|
|
||||||
)
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# FireRed-Image-Edit
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=QwenImageEditPlusSamplingParams,
|
|
||||||
pipeline_config_cls=QwenImageEditPlusPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"FireRedTeam/FireRed-Image-Edit-1.0",
|
|
||||||
"FireRedTeam/FireRed-Image-Edit-1.1",
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# ErnieImage
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=ErnieImageSamplingParams,
|
|
||||||
pipeline_config_cls=ErnieImagePipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"baidu/ERNIE-Image",
|
|
||||||
"baidu/ERNIE-Image-Turbo",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "ernie-image" in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# JoyAI
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=JoyImageEditSamplingParams,
|
|
||||||
pipeline_config_cls=JoyImageEditPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"jdopensource/JoyAI-Image-Edit-Diffusers",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "joyai-image-edit" in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=JoyEchoSamplingParams,
|
|
||||||
pipeline_config_cls=JoyEchoPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"jdopensource/JoyAI-Echo",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
("joy-echo" in hf_id.lower() or "joyai-echo" in hf_id.lower())
|
|
||||||
and "image-edit" not in hf_id.lower()
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# Ideogram 4
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Ideogram4FastSamplingParams,
|
|
||||||
pipeline_config_cls=Ideogram4DistilledPipelineConfig,
|
|
||||||
hf_model_paths=["fal/ideogram-v4-fast"],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Ideogram4InstantSamplingParams,
|
|
||||||
pipeline_config_cls=Ideogram4DistilledPipelineConfig,
|
|
||||||
hf_model_paths=["fal/ideogram-v4-instant"],
|
|
||||||
)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=Ideogram4SamplingParams,
|
|
||||||
pipeline_config_cls=Ideogram4PipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"ideogram-ai/ideogram-4-fp8",
|
|
||||||
"ideogram-ai/ideogram-4-nf4",
|
|
||||||
"Comfy-Org/Ideogram-4",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "ideogram4pipeline" in hf_id.lower(),
|
|
||||||
lambda hf_id: "ideogram-4-fp8" in hf_id.lower(),
|
|
||||||
lambda hf_id: "ideogram-4-nf4" in hf_id.lower(),
|
|
||||||
lambda hf_id: "comfy-org/ideogram-4" in hf_id.lower(),
|
|
||||||
lambda hf_id: "comfy-org--ideogram-4" in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LingBotVideoMoESamplingParams,
|
|
||||||
pipeline_config_cls=LingBotVideoMoEPipelineConfig,
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "lingbot-video-moe" in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# LongCat-Image
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LongCatImageSamplingParams,
|
|
||||||
pipeline_config_cls=LongCatImagePipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"meituan-longcat/LongCat-Image",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: "longcat" in hf_id.lower() and "edit" not in hf_id.lower(),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# LongCat-Image-Edit-Turbo (registered before Edit so its detector wins)
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LongCatImageEditTurboSamplingParams,
|
|
||||||
pipeline_config_cls=LongCatImageEditPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"meituan-longcat/LongCat-Image-Edit-Turbo",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"longcat" in hf_id.lower()
|
|
||||||
and "edit" in hf_id.lower()
|
|
||||||
and "turbo" in hf_id.lower()
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
# LongCat-Image-Edit
|
|
||||||
register_configs(
|
|
||||||
sampling_param_cls=LongCatImageEditSamplingParams,
|
|
||||||
pipeline_config_cls=LongCatImageEditPipelineConfig,
|
|
||||||
hf_model_paths=[
|
|
||||||
"meituan-longcat/LongCat-Image-Edit",
|
|
||||||
],
|
|
||||||
model_detectors=[
|
|
||||||
lambda hf_id: (
|
|
||||||
"longcat" in hf_id.lower()
|
|
||||||
and "edit" in hf_id.lower()
|
|
||||||
and "turbo" not in hf_id.lower()
|
|
||||||
),
|
|
||||||
],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
_register_configs()
|
|
||||||
|
|
||||||
|
|
||||||
def is_known_non_diffusers_multimodal_model(model_path: str) -> bool:
|
def is_known_non_diffusers_multimodal_model(model_path: str) -> bool:
|
||||||
|
|||||||
Reference in New Issue
Block a user