[Feat] diffusion: support wan-animate-2 distilled model (#44085)
Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com>
This commit is contained in:
co-authored by
Ankit Dhall
parent
9d5386fd13
commit
f727076ed0
@@ -23,7 +23,10 @@ from sglang.multimodal_gen.configs.models.dits.sana_video import SanaVideoConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.stablediffusion3 import (
|
||||
StableDiffusion3TransformerConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
|
||||
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
|
||||
WanAnimate2Config,
|
||||
WanAnimate2DistilledConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig
|
||||
|
||||
__all__ = [
|
||||
@@ -38,6 +41,7 @@ __all__ = [
|
||||
"LongLive2VideoConfig",
|
||||
"MiniMaxH3DiTConfig",
|
||||
"WanAnimate2Config",
|
||||
"WanAnimate2DistilledConfig",
|
||||
"WanVideoConfig",
|
||||
"Hunyuan3DDiTConfig",
|
||||
"MOVAAudioConfig",
|
||||
|
||||
@@ -96,8 +96,9 @@ class WanAnimate2ArchConfig(WanVideoArchConfig):
|
||||
-1
|
||||
) # <0: resolved to the gen grid width in tokens at runtime (official convention)
|
||||
refer_stride: int = 1
|
||||
# Additive attention-score bias on the keys of generation frame 1 (official name);
|
||||
# 0.0 disables it. Read from transformer/config.json when present; the base weights omit it.
|
||||
# Additive attention-score bias on the keys of generation frame 1 (official name); 0.0
|
||||
# disables it. Neither published transformer/config.json carries the key, so the distilled
|
||||
# default comes from WanAnimate2DistilledArchConfig; a checkpoint key wins via update_model_arch.
|
||||
log_scale: float = 0.0
|
||||
# The image-embedding cross-attention (added_kv_proj_dim) is always built, as in the
|
||||
# official model; the checkpoint config states the same and nothing else reads this.
|
||||
@@ -128,3 +129,17 @@ class WanAnimate2ArchConfig(WanVideoArchConfig):
|
||||
class WanAnimate2Config(DiTConfig):
|
||||
arch_config: WanAnimate2ArchConfig = field(default_factory=WanAnimate2ArchConfig)
|
||||
prefix: str = "WanAnimate2"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WanAnimate2DistilledArchConfig(WanAnimate2ArchConfig):
|
||||
# Value of the official wan_animate_2_distillation.yaml; the published transformer/config.json
|
||||
# omits the key, so a checkpoint value still wins through update_model_arch.
|
||||
log_scale: float = -1.3
|
||||
|
||||
|
||||
@dataclass
|
||||
class WanAnimate2DistilledConfig(WanAnimate2Config):
|
||||
arch_config: WanAnimate2ArchConfig = field(
|
||||
default_factory=WanAnimate2DistilledArchConfig
|
||||
)
|
||||
|
||||
@@ -2,14 +2,18 @@
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig
|
||||
from sglang.multimodal_gen.configs.models.dits import WanAnimate2Config, WanVideoConfig
|
||||
from sglang.multimodal_gen.configs.models.dits import (
|
||||
WanAnimate2Config,
|
||||
WanAnimate2DistilledConfig,
|
||||
WanVideoConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.encoders import (
|
||||
BaseEncoderOutput,
|
||||
CLIPVisionConfig,
|
||||
@@ -428,6 +432,21 @@ class Wan_Animate_2_14B_Config(WanI2V720PConfig):
|
||||
super().validate_server_args(server_args)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Wan_Animate_2_14B_Distilled_Config(Wan_Animate_2_14B_Config):
|
||||
"""Distilled Wan-Animate-2 14B: same components and stages, log_scale -1.3 in the DiT."""
|
||||
|
||||
dit_config: DiTConfig = field(default_factory=WanAnimate2DistilledConfig)
|
||||
|
||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||
# No unconditional branch to split: a CFG group would recompute one branch per rank,
|
||||
# so an explicit CFG group is rejected. Auto-enable needs no flag: the server only
|
||||
# auto-enables CFG parallel for a model whose default sampling params use guidance.
|
||||
return replace(
|
||||
super().get_model_deployment_config(), supports_cfg_parallel=False
|
||||
)
|
||||
|
||||
|
||||
# =============================================
|
||||
# ============= Causal Self-Forcing =============
|
||||
# =============================================
|
||||
@@ -441,6 +460,19 @@ class SelfForcingWanT2V480PConfig(WanT2V480PConfig):
|
||||
warp_denoising_step: bool = True
|
||||
|
||||
|
||||
def _is_wan_animate_2_path(p: str) -> bool:
|
||||
# Registry probes are lower-cased. Require the "-2-"/"_2" token so a v1
|
||||
# Wan2.2-Animate-14B checkpoint (a different model / pipeline) does not route here.
|
||||
return "wan_animate_2" in p or "wan2.2-animate-2" in p
|
||||
|
||||
|
||||
def _is_wan_animate_2_distilled_path(p: str) -> bool:
|
||||
# The distill token counts in the directory's own name only, so a base checkpoint
|
||||
# kept under a parent such as distillation-runs/ keeps the base config.
|
||||
short_name = p.rstrip("/").split("/")[-1]
|
||||
return _is_wan_animate_2_path(p) and "distill" in short_name
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.wan import (
|
||||
FastWanT2V480PConfig,
|
||||
@@ -449,6 +481,7 @@ def register():
|
||||
Wan2_2_I2V_A14B_SamplingParam,
|
||||
Wan2_2_T2V_A14B_SamplingParam,
|
||||
Wan2_2_TI2V_5B_SamplingParam,
|
||||
Wan_Animate_2_14B_Distilled_SamplingParam,
|
||||
Wan_Animate_2_14B_SamplingParam,
|
||||
WanI2V_14B_480P_SamplingParam,
|
||||
WanI2V_14B_720P_SamplingParam,
|
||||
@@ -551,20 +584,34 @@ def register():
|
||||
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
],
|
||||
)
|
||||
# Both repos ship model_index.json with the same _class_name (WanAnimate2Pipeline);
|
||||
# modular_model_index.json names the variant, and the registry probes the detectors
|
||||
# with both classes and the path (a distill token counts in the directory's own name
|
||||
# only). Detectors are checked in registration order, and a directory that ships
|
||||
# model_index.json matches the base detector through the shared class as well, so
|
||||
# the distilled entry is registered first.
|
||||
register_configs(
|
||||
sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
|
||||
pipeline_config_cls=Wan_Animate_2_14B_Config,
|
||||
hf_model_paths=[
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
|
||||
],
|
||||
sampling_param_cls=Wan_Animate_2_14B_Distilled_SamplingParam,
|
||||
pipeline_config_cls=Wan_Animate_2_14B_Distilled_Config,
|
||||
hf_model_paths=["Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"],
|
||||
model_detectors=[
|
||||
# The registry calls this with the path and with model_index's lower-cased
|
||||
# _class_name. Require the "-2-"/"_2" token so a v1 Wan2.2-Animate-14B
|
||||
# checkpoint (a different model / pipeline) does not mis-route here.
|
||||
lambda p: (
|
||||
"wan_animate_2" in p.lower()
|
||||
or "wan2.2-animate-2" in p.lower()
|
||||
or p.lower() == "wananimate2pipeline"
|
||||
p == "wananimate2distilledmodularpipeline"
|
||||
or _is_wan_animate_2_distilled_path(p)
|
||||
)
|
||||
],
|
||||
)
|
||||
register_configs(
|
||||
sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
|
||||
pipeline_config_cls=Wan_Animate_2_14B_Config,
|
||||
hf_model_paths=["Wan-AI/Wan2.2-Animate-2-14B-Diffusers"],
|
||||
model_detectors=[
|
||||
lambda p: (
|
||||
p in ("wananimate2modularpipeline", "wananimate2pipeline")
|
||||
or (
|
||||
_is_wan_animate_2_path(p)
|
||||
and not _is_wan_animate_2_distilled_path(p)
|
||||
)
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
@@ -395,6 +395,27 @@ class Wan_Animate_2_14B_SamplingParam(Wan2_2_Base_SamplingParams):
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Wan_Animate_2_14B_Distilled_SamplingParam(Wan_Animate_2_14B_SamplingParam):
|
||||
"""Sampling defaults for the distilled Wan-Animate-2 14B checkpoint (10 steps, no CFG)."""
|
||||
|
||||
# Guidance is distilled into the weights; the stage runs a single forward at 1.0.
|
||||
guidance_scale: float = 1.0
|
||||
num_inference_steps: int = 10
|
||||
|
||||
def _validate_with_pipeline_config(self, pipeline_config) -> None:
|
||||
super()._validate_with_pipeline_config(pipeline_config)
|
||||
# Runs once per request on the merged params. ValueError like the base hook's
|
||||
# input checks: the /v1/videos handler maps it to HTTP 400.
|
||||
if self.guidance_scale != 1.0:
|
||||
raise ValueError(
|
||||
f"guidance_scale={self.guidance_scale} is not supported by the "
|
||||
"distilled Wan-Animate-2 checkpoint: it samples at guidance_scale 1.0 "
|
||||
"only (the distilled recipe has no unconditional branch); omit the "
|
||||
"field or set it to 1.0."
|
||||
)
|
||||
|
||||
|
||||
# Smallest clip: one conditioning frame plus one VAE temporal stride (4k+1, k >= 1).
|
||||
MIN_CLIP_LEN = 5
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.multimodal_gen.runtime.utils.external_model_package import (
|
||||
load_external_model_package,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
load_dict,
|
||||
maybe_download_model_index,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
@@ -473,10 +474,22 @@ def _get_config_info(
|
||||
config = maybe_download_model_index(model_path)
|
||||
pipeline_name = config.get("_class_name", "")
|
||||
pipeline_name = pipeline_name.lower()
|
||||
# A local checkpoint that ships modular_model_index.json next to model_index.json
|
||||
# may name a finer-grained pipeline there (the Wan-Animate-2 variants);
|
||||
# maybe_download_model_index reads the modular file only when model_index.json is
|
||||
# absent, so probe the detectors with its class as well.
|
||||
modular_pipeline_name = (
|
||||
load_dict(os.path.join(model_path, "modular_model_index.json"))
|
||||
.get("_class_name", "")
|
||||
.lower()
|
||||
)
|
||||
probes = [model_path.lower(), pipeline_name]
|
||||
if modular_pipeline_name and modular_pipeline_name != pipeline_name:
|
||||
probes.append(modular_pipeline_name)
|
||||
|
||||
matched_model_names = []
|
||||
for model_id, detector in _MODEL_NAME_DETECTORS:
|
||||
if detector(model_path.lower()) or detector(pipeline_name):
|
||||
if any(detector(probe) for probe in probes):
|
||||
logger.debug(
|
||||
f"Matched model name '{model_id}' using a registered detector."
|
||||
)
|
||||
|
||||
@@ -9,11 +9,20 @@ standard component loaders. Text uses TextEncodingStage; CLIP and VAE adapters
|
||||
preserve the reference preprocessing and latent scaling.
|
||||
"""
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
||||
Wan_Animate_2_14B_Distilled_Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
|
||||
SchedulerLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
|
||||
TransformerLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
|
||||
DPMSolverMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
@@ -25,6 +34,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.w
|
||||
WanAnimate2VaeAdapter,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class WanAnimate2TransformerLoader(TransformerLoader):
|
||||
@@ -34,20 +46,51 @@ class WanAnimate2TransformerLoader(TransformerLoader):
|
||||
strict_checkpoint_keys = True
|
||||
|
||||
|
||||
class WanAnimate2SchedulerLoader(SchedulerLoader):
|
||||
"""Builds DPM-Solver++ 2M from the flow-matching settings for the distilled checkpoint
|
||||
and defers to the shared loader (the checkpoint's own scheduler/) for the base one;
|
||||
the cookbook page explains why the shipped scheduler config is not read."""
|
||||
|
||||
# Pipeline-local: an empty component_names keeps this subclass out of the shared
|
||||
# loader registry (ComponentLoader.__init_subclass__ registers by component name).
|
||||
component_names = []
|
||||
|
||||
def load_customized(
|
||||
self, component_model_path: str, server_args: ServerArgs, *args
|
||||
):
|
||||
pipeline_config = server_args.pipeline_config
|
||||
if isinstance(pipeline_config, Wan_Animate_2_14B_Distilled_Config):
|
||||
override = pipeline_config.scheduler_class_override
|
||||
if override:
|
||||
logger.warning(
|
||||
"--scheduler-class-override %s is ignored for the distilled "
|
||||
"Wan-Animate-2 checkpoint, which samples with DPM-Solver++ 2M on "
|
||||
"the official sigma grid; drop the flag.",
|
||||
override,
|
||||
)
|
||||
return DPMSolverMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
prediction_type="flow_prediction",
|
||||
use_flow_sigmas=True,
|
||||
flow_shift=pipeline_config.flow_shift,
|
||||
)
|
||||
return super().load_customized(component_model_path, server_args, *args)
|
||||
|
||||
|
||||
class WanAnimate2Pipeline(ComposedPipelineBase):
|
||||
"""Wan-Animate-2 14B in-context reference image + reference video -> video pipeline."""
|
||||
|
||||
pipeline_name = "WanAnimate2Pipeline"
|
||||
is_video_pipeline = True
|
||||
|
||||
# All seven model_index.json components. The scheduler is built from the
|
||||
# checkpoint's scheduler/scheduler_config.json (DPM-Solver++, flow prediction,
|
||||
# flow sigmas) by the shared loader. Two of its settings are overridden at
|
||||
# runtime: the stages call set_timesteps(sigmas=...) with the official Wan
|
||||
# sigma grid at every clip (sigma_0 is exactly 1.0), shifted by
|
||||
# pipeline_config.flow_shift, so the JSON's flow_shift and timestep_spacing
|
||||
# are not consulted. The JSON's beta_schedule only shapes the beta-derived
|
||||
# tables, which the flow-sigma path does not read.
|
||||
# All seven model_index.json components; the scheduler comes from
|
||||
# WanAnimate2SchedulerLoader (DPM-Solver++, flow prediction, flow sigmas for
|
||||
# both checkpoints). Two of its settings are overridden at runtime: the
|
||||
# stages call set_timesteps(sigmas=...) with the official Wan sigma grid at
|
||||
# every clip (sigma_0 is exactly 1.0), shifted by pipeline_config.flow_shift,
|
||||
# so the JSON's flow_shift and timestep_spacing are not consulted. The JSON's
|
||||
# beta_schedule only shapes the beta-derived tables, which the flow-sigma
|
||||
# path does not read.
|
||||
_required_config_modules: list[str] = [
|
||||
"text_encoder",
|
||||
"tokenizer",
|
||||
@@ -57,7 +100,10 @@ class WanAnimate2Pipeline(ComposedPipelineBase):
|
||||
"image_processor",
|
||||
"scheduler",
|
||||
]
|
||||
component_loaders = {"transformer": WanAnimate2TransformerLoader}
|
||||
component_loaders = {
|
||||
"transformer": WanAnimate2TransformerLoader,
|
||||
"scheduler": WanAnimate2SchedulerLoader,
|
||||
}
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
|
||||
# Keep native modules visible to residency and layerwise-offload management;
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
SANA_WM_TI2V_CI_sampling_params,
|
||||
T2I_sampling_params,
|
||||
T2V_sampling_params,
|
||||
WAN_2_2_ANIMATE_2_14B_DISTILLED_sampling_params,
|
||||
WAN_2_2_ANIMATE_2_14B_sampling_params,
|
||||
_make_modelopt_ci_case,
|
||||
_with_default_num_gpus,
|
||||
@@ -64,6 +65,7 @@ from sglang.multimodal_gen.test.test_utils import (
|
||||
DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_ANIMATE_2_14B_DISTILLED_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
|
||||
@@ -478,6 +480,21 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
run_perf_check=True,
|
||||
run_component_accuracy_check=False,
|
||||
),
|
||||
# Distilled checkpoint on the same inputs and server layout: 10 steps at guidance
|
||||
# 1.0, so the denoiser runs the single-forward path (no unconditional batch).
|
||||
DiffusionTestCase(
|
||||
"wan2_2_animate_2_14b_distilled_ref2v",
|
||||
DiffusionServerArgs(
|
||||
model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_DISTILLED_MODEL_NAME_FOR_TEST,
|
||||
modality="video",
|
||||
dit_layerwise_offload=True,
|
||||
text_encoder_cpu_offload=True,
|
||||
extras=["--layerwise-offload-components", "transformer"],
|
||||
),
|
||||
WAN_2_2_ANIMATE_2_14B_DISTILLED_sampling_params,
|
||||
run_perf_check=True,
|
||||
run_component_accuracy_check=False,
|
||||
),
|
||||
# flaky
|
||||
# === Helios T2V ===
|
||||
# DiffusionTestCase(
|
||||
|
||||
@@ -3561,6 +3561,35 @@
|
||||
"estimated_full_test_time_s": 489.9,
|
||||
"warmup_peak_vram_mb": 14598.0
|
||||
},
|
||||
"wan2_2_animate_2_14b_distilled_ref2v": {
|
||||
"stages_ms": {
|
||||
"WanAnimate2BeforeDenoisingStage": 7846.04,
|
||||
"WanAnimate2DenoisingStage": 69260.73,
|
||||
"WanAnimate2OutputStage": 0.08
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 23165.59,
|
||||
"1": 3081.14,
|
||||
"2": 4366.18,
|
||||
"3": 4368.35,
|
||||
"4": 4373.6,
|
||||
"5": 4377.11,
|
||||
"6": 4376.38,
|
||||
"7": 4380.89,
|
||||
"8": 4382.1,
|
||||
"9": 4374.26
|
||||
},
|
||||
"expected_e2e_ms": 76924.33,
|
||||
"expected_load_ms": 48892.73,
|
||||
"expected_avg_denoise_ms": 6117.67,
|
||||
"expected_median_denoise_ms": 4375.32,
|
||||
"load_peak_vram_mb": 1776.0,
|
||||
"runtime_peak_vram_mb": 36322.0,
|
||||
"load_peak_allocated_mb": 1737.0,
|
||||
"runtime_peak_allocated_mb": 25748.0,
|
||||
"estimated_full_test_time_s": 439.9,
|
||||
"warmup_peak_vram_mb": 14598.0
|
||||
},
|
||||
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
|
||||
"stages_ms": {
|
||||
"WanAnimate2BeforeDenoisingStage": 5778.62,
|
||||
|
||||
@@ -939,6 +939,17 @@ WAN_2_2_ANIMATE_2_14B_sampling_params = DiffusionSamplingParams(
|
||||
},
|
||||
)
|
||||
|
||||
# Distilled checkpoint: guidance is folded into the weights, so the request runs the
|
||||
# model's 10-step, guidance-1.0 recipe over the same inputs, canvas and clip length.
|
||||
WAN_2_2_ANIMATE_2_14B_DISTILLED_sampling_params = replace(
|
||||
WAN_2_2_ANIMATE_2_14B_sampling_params,
|
||||
extras={
|
||||
**WAN_2_2_ANIMATE_2_14B_sampling_params.extras,
|
||||
"num_inference_steps": 10,
|
||||
"guidance_scale": 1.0,
|
||||
},
|
||||
)
|
||||
|
||||
HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams(
|
||||
prompt="",
|
||||
image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png",
|
||||
|
||||
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
|
||||
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
|
||||
# publish.
|
||||
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "75992dbf1d6eaeacbc1c2cd6beb48f77dd7ae4f7"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "252df4958ae6cc9f6b146b2f70382f75459235ad"
|
||||
|
||||
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
|
||||
# when it's regenerated on its own cadence. 2173af2f re-generated the MiniMax-H3
|
||||
@@ -208,6 +208,9 @@ DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-I2V-A14B-Diffusers
|
||||
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST = (
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
|
||||
)
|
||||
DEFAULT_WAN_2_2_ANIMATE_2_14B_DISTILLED_MODEL_NAME_FOR_TEST = (
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"
|
||||
)
|
||||
|
||||
# MOVA video generation models
|
||||
DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST = "OpenMOSS-Team/MOVA-360p"
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -78,3 +80,58 @@ def default_global_server_args():
|
||||
yield
|
||||
finally:
|
||||
set_global_server_args(previous)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def mock_cuda_platform(
|
||||
*,
|
||||
memory_gb: int = 80,
|
||||
available_memory_gb: int | dict[int, int] | None = None,
|
||||
):
|
||||
"""current_platform as an idle CUDA device of memory_gb GB, so ServerArgs resolution
|
||||
and parallelism auto-tuning run without a GPU."""
|
||||
|
||||
def get_available_gpu_memory(device_id=0, **_kwargs):
|
||||
if isinstance(available_memory_gb, dict):
|
||||
return available_memory_gb[device_id]
|
||||
if available_memory_gb is not None:
|
||||
return available_memory_gb
|
||||
return memory_gb
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_mps",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_rocm",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_total_memory",
|
||||
return_value=memory_gb * 1024**3,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_available_gpu_memory",
|
||||
side_effect=get_available_gpu_memory,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cuda_platform():
|
||||
with mock_cuda_platform():
|
||||
yield
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Model routing in the registry, without Hub access."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen import registry
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
||||
Wan_Animate_2_14B_Config,
|
||||
Wan_Animate_2_14B_Distilled_Config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_registry_cache():
|
||||
registry.get_model_info.cache_clear()
|
||||
registry._get_config_info.cache_clear()
|
||||
yield
|
||||
registry.get_model_info.cache_clear()
|
||||
registry._get_config_info.cache_clear()
|
||||
|
||||
|
||||
def _checkpoint_dir(
|
||||
root,
|
||||
*,
|
||||
name: str = "ckpt-7f3a",
|
||||
index_class: str = "SomePipeline",
|
||||
modular_class: str | None = None,
|
||||
) -> str:
|
||||
"""Laid out like a published repo: model_index.json, the component folders the index
|
||||
verifier looks for, and modular_model_index.json when a class is given."""
|
||||
ckpt = root / name
|
||||
ckpt.mkdir(parents=True)
|
||||
(ckpt / "transformer").mkdir()
|
||||
(ckpt / "vae").mkdir()
|
||||
index = {"_class_name": index_class, "_diffusers_version": "0.40.0.dev0"}
|
||||
(ckpt / "model_index.json").write_text(json.dumps(index))
|
||||
if modular_class is not None:
|
||||
(ckpt / "modular_model_index.json").write_text(
|
||||
json.dumps({**index, "_class_name": modular_class})
|
||||
)
|
||||
return str(ckpt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("modular_class", [None, "SomeModularPipeline"])
|
||||
def test_detectors_are_probed_with_the_modular_pipeline_class(
|
||||
tmp_path, monkeypatch, modular_class
|
||||
):
|
||||
# A checkpoint may ship modular_model_index.json next to model_index.json and name
|
||||
# a finer-grained pipeline there; without it the probes are the path and the
|
||||
# model_index class only.
|
||||
probes = []
|
||||
|
||||
def detector(probe: str) -> bool:
|
||||
probes.append(probe)
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(registry, "_MODEL_NAME_DETECTORS", [("probe", detector)])
|
||||
path = _checkpoint_dir(tmp_path, modular_class=modular_class)
|
||||
assert registry._get_config_info(path) is None
|
||||
expected = [path.lower(), "somepipeline"]
|
||||
if modular_class is not None:
|
||||
expected.append(modular_class.lower())
|
||||
assert probes == expected
|
||||
|
||||
|
||||
# Wan-Animate-2: both repos ship model_index.json with the same _class_name;
|
||||
# modular_model_index.json names the variant, and a distill token in the directory's
|
||||
# own name selects the distilled entry.
|
||||
|
||||
_WAN_ANIMATE_2 = "WanAnimate2Pipeline"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"modular_class, config_cls",
|
||||
[
|
||||
("WanAnimate2ModularPipeline", Wan_Animate_2_14B_Config),
|
||||
("WanAnimate2DistilledModularPipeline", Wan_Animate_2_14B_Distilled_Config),
|
||||
# A mirror that skipped modular_model_index.json leaves only the shared class.
|
||||
(None, Wan_Animate_2_14B_Config),
|
||||
],
|
||||
)
|
||||
def test_neutral_directory_routes_by_its_modular_pipeline_class(
|
||||
tmp_path, modular_class, config_cls
|
||||
):
|
||||
path = _checkpoint_dir(
|
||||
tmp_path, index_class=_WAN_ANIMATE_2, modular_class=modular_class
|
||||
)
|
||||
assert registry._get_config_info(path).pipeline_config_cls is config_cls
|
||||
|
||||
|
||||
def test_distill_token_in_the_directory_name_routes_to_the_distilled_config(tmp_path):
|
||||
# The token routes without the modular index; the shared model_index class matches
|
||||
# the base detector too, so the distilled entry is registered first.
|
||||
path = _checkpoint_dir(
|
||||
tmp_path, name="wan_animate_2_distilled", index_class=_WAN_ANIMATE_2
|
||||
)
|
||||
info = registry._get_config_info(path)
|
||||
assert info.pipeline_config_cls is Wan_Animate_2_14B_Distilled_Config
|
||||
|
||||
|
||||
def test_distill_token_in_a_parent_directory_leaves_the_checkpoint_in_charge(tmp_path):
|
||||
# Only the directory's own name counts: a base checkpoint kept under a parent such
|
||||
# as distillation-runs/ routes by its modular class.
|
||||
path = _checkpoint_dir(
|
||||
tmp_path / "distillation-runs",
|
||||
name="wan_animate_2_base",
|
||||
index_class=_WAN_ANIMATE_2,
|
||||
modular_class="WanAnimate2ModularPipeline",
|
||||
)
|
||||
assert (
|
||||
registry._get_config_info(path).pipeline_config_cls is Wan_Animate_2_14B_Config
|
||||
)
|
||||
|
||||
|
||||
def test_model_id_short_name_selects_the_variant_for_a_neutral_directory():
|
||||
info = registry._get_config_info(
|
||||
"/models/ckpt-7f3a", model_id="Wan2.2-Animate-2-14B-Distilled-Diffusers"
|
||||
)
|
||||
assert info.pipeline_config_cls is Wan_Animate_2_14B_Distilled_Config
|
||||
@@ -4,7 +4,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from contextlib import ExitStack
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import zmq
|
||||
@@ -97,52 +97,7 @@ from sglang.multimodal_gen.runtime.server_args import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import server_args as server_args_module
|
||||
from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _mock_cuda_platform(
|
||||
*,
|
||||
memory_gb: int = 80,
|
||||
available_memory_gb: int | dict[int, int] | None = None,
|
||||
):
|
||||
def get_available_gpu_memory(device_id=0, **_kwargs):
|
||||
if isinstance(available_memory_gb, dict):
|
||||
return available_memory_gb[device_id]
|
||||
if available_memory_gb is not None:
|
||||
return available_memory_gb
|
||||
return memory_gb
|
||||
|
||||
with (
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_mps",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
|
||||
return_value=True,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_rocm",
|
||||
return_value=False,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_total_memory",
|
||||
return_value=memory_gb * 1024**3,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.get_available_gpu_memory",
|
||||
side_effect=get_available_gpu_memory,
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
|
||||
return_value=True,
|
||||
),
|
||||
):
|
||||
yield
|
||||
from sglang.multimodal_gen.test.unit.conftest import mock_cuda_platform
|
||||
|
||||
|
||||
def _from_dict_without_model_resolution(
|
||||
@@ -151,7 +106,7 @@ def _from_dict_without_model_resolution(
|
||||
pipeline_config = pipeline_config or QwenImagePipelineConfig()
|
||||
with (
|
||||
patch.object(PipelineConfig, "from_kwargs", return_value=pipeline_config),
|
||||
_mock_cuda_platform(),
|
||||
mock_cuda_platform(),
|
||||
):
|
||||
return ServerArgs.from_dict(kwargs)
|
||||
|
||||
@@ -191,7 +146,7 @@ class _CudaPlatformTestCase(unittest.TestCase):
|
||||
# Residency queries inspect the platform after argument construction too.
|
||||
stack = ExitStack()
|
||||
self.addCleanup(stack.close)
|
||||
stack.enter_context(_mock_cuda_platform())
|
||||
stack.enter_context(mock_cuda_platform())
|
||||
|
||||
|
||||
class TestSchedulerEndpoints(unittest.TestCase):
|
||||
@@ -284,7 +239,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
|
||||
patch.object(
|
||||
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
|
||||
),
|
||||
_mock_cuda_platform(),
|
||||
mock_cuda_platform(),
|
||||
):
|
||||
args, unknown_args = parser.parse_known_args(argv)
|
||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
||||
@@ -570,7 +525,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
|
||||
patch.object(
|
||||
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
|
||||
),
|
||||
_mock_cuda_platform(),
|
||||
mock_cuda_platform(),
|
||||
):
|
||||
args, unknown_args = parser.parse_known_args(argv)
|
||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
||||
@@ -591,7 +546,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
|
||||
patch.object(
|
||||
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
|
||||
),
|
||||
_mock_cuda_platform(),
|
||||
mock_cuda_platform(),
|
||||
):
|
||||
args, unknown_args = parser.parse_known_args(argv)
|
||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
||||
@@ -3550,7 +3505,7 @@ class TestPipelineResolutionCliOverride(unittest.TestCase):
|
||||
|
||||
with (
|
||||
patch.object(sys, "argv", ["sglang"] + argv),
|
||||
_mock_cuda_platform(),
|
||||
mock_cuda_platform(),
|
||||
):
|
||||
args, unknown_args = parser.parse_known_args(argv)
|
||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
||||
@@ -3569,7 +3524,7 @@ class TestPipelineResolutionCliOverride(unittest.TestCase):
|
||||
|
||||
with (
|
||||
patch.object(sys, "argv", ["sglang"] + argv),
|
||||
_mock_cuda_platform(),
|
||||
mock_cuda_platform(),
|
||||
):
|
||||
args, unknown_args = parser.parse_known_args(argv)
|
||||
server_args = ServerArgs.from_cli_args(args, unknown_args)
|
||||
|
||||
@@ -23,12 +23,19 @@ from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
|
||||
WanAnimate2ArchConfig,
|
||||
WanAnimate2Config,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan_Animate_2_14B_Config
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
||||
Wan_Animate_2_14B_Config,
|
||||
Wan_Animate_2_14B_Distilled_Config,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.wan import Wan_Animate_2_14B_SamplingParam
|
||||
from sglang.multimodal_gen.configs.sample.wan import (
|
||||
Wan_Animate_2_14B_Distilled_SamplingParam,
|
||||
Wan_Animate_2_14B_SamplingParam,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import _get_config_info
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
@@ -46,6 +53,7 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend i
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
|
||||
SchedulerLoader,
|
||||
_supported_init_kwargs,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
|
||||
@@ -67,6 +75,7 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multi
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.wan_animate_2_pipeline import (
|
||||
WanAnimate2Pipeline,
|
||||
WanAnimate2SchedulerLoader,
|
||||
WanAnimate2TransformerLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
@@ -117,8 +126,12 @@ def test_pipeline_local_loader_preserves_shared_transformer_registry():
|
||||
assert component_name_to_loader_cls[name] is TransformerLoader
|
||||
assert not TransformerLoader.strict_checkpoint_keys
|
||||
assert WanAnimate2TransformerLoader.strict_checkpoint_keys
|
||||
for name in SchedulerLoader.component_names:
|
||||
assert component_name_to_loader_cls[name] is SchedulerLoader
|
||||
assert WanAnimate2SchedulerLoader.component_names == []
|
||||
assert WanAnimate2Pipeline.component_loaders == {
|
||||
"transformer": WanAnimate2TransformerLoader
|
||||
"transformer": WanAnimate2TransformerLoader,
|
||||
"scheduler": WanAnimate2SchedulerLoader,
|
||||
}
|
||||
|
||||
|
||||
@@ -241,6 +254,57 @@ def _parallel_server_args(
|
||||
)
|
||||
|
||||
|
||||
def _two_gpu_server_args(pipeline_config, **kwargs) -> ServerArgs:
|
||||
"""ServerArgs for the config's HF id with the config pinned and model_index.json
|
||||
served from memory; under the cuda_platform fixture _adjust_parallelism runs
|
||||
without a GPU or network."""
|
||||
hf_id = "Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
|
||||
if isinstance(pipeline_config, Wan_Animate_2_14B_Distilled_Config):
|
||||
hf_id = "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"
|
||||
with (
|
||||
patch.object(PipelineConfig, "from_kwargs", return_value=pipeline_config),
|
||||
patch(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
return_value={"_class_name": "WanAnimate2Pipeline"},
|
||||
),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.auto_tune.dit_parameter_count",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
return ServerArgs.from_dict({"model_path": hf_id, "num_gpus": 2, **kwargs})
|
||||
|
||||
|
||||
def test_two_gpu_default_layout_splits_cfg_only_for_the_base_checkpoint(
|
||||
cuda_platform,
|
||||
):
|
||||
# With no parallelism flags the server auto-enables CFG parallel for a model whose
|
||||
# defaults use guidance; the distilled checkpoint runs one branch, so its second GPU
|
||||
# goes to sequence parallelism instead.
|
||||
base = _two_gpu_server_args(Wan_Animate_2_14B_Config())
|
||||
distilled = _two_gpu_server_args(Wan_Animate_2_14B_Distilled_Config())
|
||||
assert base.enable_cfg_parallel and not distilled.enable_cfg_parallel
|
||||
assert (base.cfg_parallel_degree, base.sp_degree) == (2, 1)
|
||||
assert (distilled.cfg_parallel_degree, distilled.sp_degree) == (1, 2)
|
||||
|
||||
|
||||
def test_distilled_rejects_explicit_cfg_parallel(cuda_platform):
|
||||
# An explicit CFG group would silently run the single branch on every rank.
|
||||
with pytest.raises(ValueError, match="does not support CFG parallelism"):
|
||||
_two_gpu_server_args(
|
||||
Wan_Animate_2_14B_Distilled_Config(), enable_cfg_parallel=True
|
||||
)
|
||||
|
||||
|
||||
def test_distilled_deployment_config_keeps_the_base_residency_policy():
|
||||
# Only the CFG-parallel field differs; a change to the base residency policy must
|
||||
# reach the distilled variant.
|
||||
base = Wan_Animate_2_14B_Config().get_model_deployment_config()
|
||||
distilled = Wan_Animate_2_14B_Distilled_Config().get_model_deployment_config()
|
||||
assert distilled.supports_cfg_parallel is False
|
||||
assert replace(distilled, supports_cfg_parallel=True) == base
|
||||
|
||||
|
||||
def test_validate_server_args_rejects_fsdp_before_any_weights_load():
|
||||
with pytest.raises(NotImplementedError, match="use-fsdp-inference"):
|
||||
Wan_Animate_2_14B_Config().validate_server_args(
|
||||
@@ -487,6 +551,16 @@ def test_checkpoint_config_without_image_embedding_is_rejected():
|
||||
config.update_model_arch({"use_img_emb": False})
|
||||
|
||||
|
||||
def test_distilled_log_scale_defaults_to_the_official_value_unless_the_checkpoint_sets_it():
|
||||
# -1.3 is the official wan_animate_2_distillation.yaml value; the published
|
||||
# transformer/config.json has no log_scale key, so this default is what runs, and a
|
||||
# checkpoint that does carry the key wins over it.
|
||||
config = Wan_Animate_2_14B_Distilled_Config().dit_config
|
||||
assert config.log_scale == -1.3
|
||||
config.update_model_arch({"log_scale": -0.7})
|
||||
assert config.log_scale == -0.7
|
||||
|
||||
|
||||
# Wan_Animate_2_14B_SamplingParam
|
||||
|
||||
|
||||
@@ -580,20 +654,91 @@ def test_sampling_params_reject_the_shared_dit_cache_heuristics(field_name):
|
||||
Wan_Animate_2_14B_SamplingParam(**{field_name: True})
|
||||
|
||||
|
||||
def _distilled_request(**user_kwargs) -> SamplingParams:
|
||||
"""A /v1/videos-style request through the public entry point: registry defaults for
|
||||
the distilled HF id merged with the user fields, adjusted and validated."""
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_class_name=None,
|
||||
backend="sglang",
|
||||
model_id=None,
|
||||
pipeline_config=Wan_Animate_2_14B_Distilled_Config(),
|
||||
output_path=None,
|
||||
num_gpus=1,
|
||||
comfyui_mode=False,
|
||||
)
|
||||
with patch(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
return_value={"_class_name": "WanAnimate2Pipeline"},
|
||||
):
|
||||
return SamplingParams.from_user_sampling_params_args(
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
|
||||
server_args,
|
||||
prompt="a person",
|
||||
image_path="reference.png",
|
||||
video_path="driving.mp4",
|
||||
**user_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"guidance_scale, rejected",
|
||||
[(3.0, True), (0.5, True), (1.0, False), (None, False)],
|
||||
)
|
||||
def test_distilled_guidance_other_than_one_is_rejected(guidance_scale, rejected):
|
||||
# The distilled recipe has no unconditional branch: the once-per-request validation
|
||||
# hook raises ValueError (HTTP 400 on /v1/videos) for every guidance_scale but 1.0;
|
||||
# an omitted field (None) takes the class default.
|
||||
user_kwargs = {} if guidance_scale is None else {"guidance_scale": guidance_scale}
|
||||
if rejected:
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=rf"guidance_scale={guidance_scale} .*guidance_scale 1\.0 only",
|
||||
):
|
||||
_distilled_request(**user_kwargs)
|
||||
return
|
||||
params = _distilled_request(**user_kwargs)
|
||||
assert params.guidance_scale == 1.0
|
||||
|
||||
|
||||
def test_distilled_sampling_defaults_run_without_cfg():
|
||||
# The official distilled recipe: 10 steps, guidance 1.0; the rest is inherited.
|
||||
params = Wan_Animate_2_14B_Distilled_SamplingParam()
|
||||
assert params.guidance_scale == 1.0
|
||||
assert params.num_inference_steps == 10
|
||||
|
||||
|
||||
# Registry resolution (no network)
|
||||
|
||||
|
||||
def test_registered_hf_path_resolves_to_wan_animate_configs():
|
||||
info = _get_config_info("Wan-AI/Wan2.2-Animate-2-14B-Diffusers")
|
||||
@pytest.mark.parametrize(
|
||||
"hf_id, pipeline_cls, sampling_cls",
|
||||
[
|
||||
(
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
|
||||
Wan_Animate_2_14B_Config,
|
||||
Wan_Animate_2_14B_SamplingParam,
|
||||
),
|
||||
(
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
|
||||
Wan_Animate_2_14B_Distilled_Config,
|
||||
Wan_Animate_2_14B_Distilled_SamplingParam,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_registered_hf_path_resolves_to_wan_animate_configs(
|
||||
hf_id, pipeline_cls, sampling_cls
|
||||
):
|
||||
info = _get_config_info(hf_id)
|
||||
assert info is not None
|
||||
assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
|
||||
assert info.sampling_param_cls is Wan_Animate_2_14B_SamplingParam
|
||||
assert info.pipeline_config_cls is pipeline_cls
|
||||
assert info.sampling_param_cls is sampling_cls
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["some/local/dir/wan_animate_2", "my-wan2.2-animate-2-ckpt"]
|
||||
)
|
||||
def test_detector_resolves_wan_animate_style_paths(path):
|
||||
# No variant token in the name: the base classes, not the distilled ones.
|
||||
with patch(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
return_value={},
|
||||
@@ -606,7 +751,8 @@ def test_detector_resolves_wan_animate_style_paths(path):
|
||||
|
||||
def test_model_index_class_name_routes_a_checkpoint_at_any_path():
|
||||
# A local checkpoint directory need not carry a model token in its name; its
|
||||
# model_index.json _class_name is what identifies it.
|
||||
# model_index.json _class_name is what identifies it. Both variants share that
|
||||
# name; modular_model_index.json tells them apart (below).
|
||||
with patch(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
return_value={"_class_name": "WanAnimate2Pipeline"},
|
||||
@@ -616,15 +762,22 @@ def test_model_index_class_name_routes_a_checkpoint_at_any_path():
|
||||
assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
|
||||
|
||||
|
||||
def test_v1_animate_checkpoint_does_not_route_to_wan_animate_2():
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["Wan-AI/Wan2.2-Animate-14B", "ckpts/Wan2.2-Animate-14B-distill"]
|
||||
)
|
||||
def test_v1_animate_checkpoint_does_not_route_to_wan_animate_2(path):
|
||||
# Wan2.2-Animate-14B (v1) is a different model and pipeline; a detector loosened to
|
||||
# "animate" would pass the positive cases above and mis-route it here.
|
||||
# "animate" would pass the positive cases above and mis-route it here, and the
|
||||
# distilled detector needs the "-2-"/"_2" token as well as "distill".
|
||||
with patch(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
return_value={"_class_name": "WanAnimatePipeline"},
|
||||
):
|
||||
info = _get_config_info("Wan-AI/Wan2.2-Animate-14B")
|
||||
assert info is None or info.pipeline_config_cls is not Wan_Animate_2_14B_Config
|
||||
info = _get_config_info(path)
|
||||
assert info is None or info.pipeline_config_cls not in (
|
||||
Wan_Animate_2_14B_Config,
|
||||
Wan_Animate_2_14B_Distilled_Config,
|
||||
)
|
||||
|
||||
|
||||
# /v1/videos routing: the model-specific knobs are declared extras, the reference image and
|
||||
@@ -1305,8 +1458,10 @@ def test_extract_inputs_rejects_batched_single_value_fields(overrides, message):
|
||||
_WanAnimate2Inputs.from_request(batch)
|
||||
|
||||
|
||||
# Scheduler: loaded from the checkpoint's scheduler/scheduler_config.json through the shared
|
||||
# SchedulerLoader like the other components. The stages pass the official sigma grid to
|
||||
# Scheduler: a loaded component like the others, through WanAnimate2SchedulerLoader. The
|
||||
# base checkpoint's scheduler/scheduler_config.json is loaded as is; for the distilled
|
||||
# config the loader builds DPMSolverMultistepScheduler from the flow-matching settings
|
||||
# instead of from its Euler scheduler_config.json. The stages pass the official sigma grid to
|
||||
# set_timesteps explicitly, so the loaded configuration must denoise exactly like a
|
||||
# DPMSolverMultistepScheduler built from the flow-matching settings alone.
|
||||
|
||||
@@ -1355,6 +1510,67 @@ _FLOW_MATCHING_SCHEDULER_KWARGS = {
|
||||
}
|
||||
|
||||
|
||||
def _load_scheduler(pipeline_config, checkpoint_config: dict | None):
|
||||
"""Run WanAnimate2SchedulerLoader with the checkpoint's scheduler_config.json served
|
||||
from memory (None: reading it is an error) and the flag-free server args it reads."""
|
||||
reader = {"return_value": dict(checkpoint_config)} if checkpoint_config else {}
|
||||
server_args = SimpleNamespace(pipeline_config=pipeline_config)
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader"
|
||||
".get_diffusers_component_config",
|
||||
**reader,
|
||||
) as read:
|
||||
scheduler = WanAnimate2SchedulerLoader().load_customized(
|
||||
"/ckpt/scheduler", server_args
|
||||
)
|
||||
assert read.called is (checkpoint_config is not None)
|
||||
return scheduler
|
||||
|
||||
|
||||
def test_base_config_loads_the_checkpoint_scheduler_as_is():
|
||||
# The shared loader path: the class the checkpoint names, built from its own settings
|
||||
# (beta_schedule=linear comes only from that file).
|
||||
scheduler = _load_scheduler(
|
||||
Wan_Animate_2_14B_Config(),
|
||||
{"_class_name": "DPMSolverMultistepScheduler", **_CHECKPOINT_SCHEDULER_CONFIG},
|
||||
)
|
||||
assert isinstance(scheduler, DPMSolverMultistepScheduler)
|
||||
assert scheduler._inner.config.beta_schedule == "linear"
|
||||
assert scheduler._inner.config.flow_shift == 5.0
|
||||
|
||||
|
||||
def test_distilled_config_gets_dpm_solver_built_from_the_flow_matching_settings():
|
||||
# The wrapper comes from the flow-matching settings and the configured flow_shift;
|
||||
# the shipped scheduler_config.json, which carries none of them, is never read.
|
||||
config = Wan_Animate_2_14B_Distilled_Config()
|
||||
config.update_config_from_dict({"flow_shift": 3.0})
|
||||
scheduler = _load_scheduler(config, None)
|
||||
assert isinstance(scheduler, DPMSolverMultistepScheduler)
|
||||
inner = scheduler._inner.config
|
||||
assert inner.algorithm_type == "dpmsolver++"
|
||||
assert inner.solver_order == 2
|
||||
for key, value in {**_FLOW_MATCHING_SCHEDULER_KWARGS, "flow_shift": 3.0}.items():
|
||||
assert getattr(inner, key) == value, key
|
||||
|
||||
|
||||
def test_distilled_config_ignores_the_scheduler_class_override_with_one_warning(caplog):
|
||||
# The flag would select the shipped Euler scheduler on the base path; the distilled
|
||||
# path keeps DPM-Solver++ and says so instead of silently ignoring the flag.
|
||||
config = Wan_Animate_2_14B_Distilled_Config()
|
||||
config.scheduler_class_override = "FlowMatchEulerDiscreteScheduler"
|
||||
with caplog.at_level(logging.WARNING):
|
||||
scheduler = _load_scheduler(config, None)
|
||||
assert isinstance(scheduler, DPMSolverMultistepScheduler)
|
||||
inner = scheduler._inner.config
|
||||
assert inner.algorithm_type == "dpmsolver++"
|
||||
for key, value in _FLOW_MATCHING_SCHEDULER_KWARGS.items():
|
||||
assert getattr(inner, key) == value, key
|
||||
warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
|
||||
assert len(warnings) == 1, warnings
|
||||
assert "--scheduler-class-override FlowMatchEulerDiscreteScheduler" in warnings[0]
|
||||
assert "ignored for the distilled Wan-Animate-2 checkpoint" in warnings[0]
|
||||
|
||||
|
||||
def test_loader_keeps_the_checkpoint_scheduler_settings():
|
||||
kwargs = _supported_init_kwargs(
|
||||
DPMSolverMultistepScheduler, dict(_CHECKPOINT_SCHEDULER_CONFIG)
|
||||
|
||||
Reference in New Issue
Block a user