[Feat] diffusion: support wan-animate-2 distilled model (#44085)

Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com>
This commit is contained in:
Ankit Dhall
2026-10-22 15:37:08 +08:00
committed by GitHub
co-authored by Ankit Dhall
parent 9d5386fd13
commit f727076ed0
18 changed files with 905 additions and 136 deletions
@@ -23,7 +23,10 @@ from sglang.multimodal_gen.configs.models.dits.sana_video import SanaVideoConfig
from sglang.multimodal_gen.configs.models.dits.stablediffusion3 import (
StableDiffusion3TransformerConfig,
)
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
WanAnimate2Config,
WanAnimate2DistilledConfig,
)
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig
__all__ = [
@@ -38,6 +41,7 @@ __all__ = [
"LongLive2VideoConfig",
"MiniMaxH3DiTConfig",
"WanAnimate2Config",
"WanAnimate2DistilledConfig",
"WanVideoConfig",
"Hunyuan3DDiTConfig",
"MOVAAudioConfig",
@@ -96,8 +96,9 @@ class WanAnimate2ArchConfig(WanVideoArchConfig):
-1
) # <0: resolved to the gen grid width in tokens at runtime (official convention)
refer_stride: int = 1
# Additive attention-score bias on the keys of generation frame 1 (official name);
# 0.0 disables it. Read from transformer/config.json when present; the base weights omit it.
# Additive attention-score bias on the keys of generation frame 1 (official name); 0.0
# disables it. Neither published transformer/config.json carries the key, so the distilled
# default comes from WanAnimate2DistilledArchConfig; a checkpoint key wins via update_model_arch.
log_scale: float = 0.0
# The image-embedding cross-attention (added_kv_proj_dim) is always built, as in the
# official model; the checkpoint config states the same and nothing else reads this.
@@ -128,3 +129,17 @@ class WanAnimate2ArchConfig(WanVideoArchConfig):
class WanAnimate2Config(DiTConfig):
arch_config: WanAnimate2ArchConfig = field(default_factory=WanAnimate2ArchConfig)
prefix: str = "WanAnimate2"
@dataclass
class WanAnimate2DistilledArchConfig(WanAnimate2ArchConfig):
# Value of the official wan_animate_2_distillation.yaml; the published transformer/config.json
# omits the key, so a checkpoint value still wins through update_model_arch.
log_scale: float = -1.3
@dataclass
class WanAnimate2DistilledConfig(WanAnimate2Config):
arch_config: WanAnimate2ArchConfig = field(
default_factory=WanAnimate2DistilledArchConfig
)
@@ -2,14 +2,18 @@
# SPDX-License-Identifier: Apache-2.0
from collections.abc import Callable
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from typing import Any
import torch
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig
from sglang.multimodal_gen.configs.models.dits import WanAnimate2Config, WanVideoConfig
from sglang.multimodal_gen.configs.models.dits import (
WanAnimate2Config,
WanAnimate2DistilledConfig,
WanVideoConfig,
)
from sglang.multimodal_gen.configs.models.encoders import (
BaseEncoderOutput,
CLIPVisionConfig,
@@ -428,6 +432,21 @@ class Wan_Animate_2_14B_Config(WanI2V720PConfig):
super().validate_server_args(server_args)
@dataclass
class Wan_Animate_2_14B_Distilled_Config(Wan_Animate_2_14B_Config):
"""Distilled Wan-Animate-2 14B: same components and stages, log_scale -1.3 in the DiT."""
dit_config: DiTConfig = field(default_factory=WanAnimate2DistilledConfig)
def get_model_deployment_config(self) -> ModelDeploymentConfig:
# No unconditional branch to split: a CFG group would recompute one branch per rank,
# so an explicit CFG group is rejected. Auto-enable needs no flag: the server only
# auto-enables CFG parallel for a model whose default sampling params use guidance.
return replace(
super().get_model_deployment_config(), supports_cfg_parallel=False
)
# =============================================
# ============= Causal Self-Forcing =============
# =============================================
@@ -441,6 +460,19 @@ class SelfForcingWanT2V480PConfig(WanT2V480PConfig):
warp_denoising_step: bool = True
def _is_wan_animate_2_path(p: str) -> bool:
# Registry probes are lower-cased. Require the "-2-"/"_2" token so a v1
# Wan2.2-Animate-14B checkpoint (a different model / pipeline) does not route here.
return "wan_animate_2" in p or "wan2.2-animate-2" in p
def _is_wan_animate_2_distilled_path(p: str) -> bool:
# The distill token counts in the directory's own name only, so a base checkpoint
# kept under a parent such as distillation-runs/ keeps the base config.
short_name = p.rstrip("/").split("/")[-1]
return _is_wan_animate_2_path(p) and "distill" in short_name
def register():
from sglang.multimodal_gen.configs.sample.wan import (
FastWanT2V480PConfig,
@@ -449,6 +481,7 @@ def register():
Wan2_2_I2V_A14B_SamplingParam,
Wan2_2_T2V_A14B_SamplingParam,
Wan2_2_TI2V_5B_SamplingParam,
Wan_Animate_2_14B_Distilled_SamplingParam,
Wan_Animate_2_14B_SamplingParam,
WanI2V_14B_480P_SamplingParam,
WanI2V_14B_720P_SamplingParam,
@@ -551,20 +584,34 @@ def register():
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
],
)
# Both repos ship model_index.json with the same _class_name (WanAnimate2Pipeline);
# modular_model_index.json names the variant, and the registry probes the detectors
# with both classes and the path (a distill token counts in the directory's own name
# only). Detectors are checked in registration order, and a directory that ships
# model_index.json matches the base detector through the shared class as well, so
# the distilled entry is registered first.
register_configs(
sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
pipeline_config_cls=Wan_Animate_2_14B_Config,
hf_model_paths=[
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
],
sampling_param_cls=Wan_Animate_2_14B_Distilled_SamplingParam,
pipeline_config_cls=Wan_Animate_2_14B_Distilled_Config,
hf_model_paths=["Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"],
model_detectors=[
# The registry calls this with the path and with model_index's lower-cased
# _class_name. Require the "-2-"/"_2" token so a v1 Wan2.2-Animate-14B
# checkpoint (a different model / pipeline) does not mis-route here.
lambda p: (
"wan_animate_2" in p.lower()
or "wan2.2-animate-2" in p.lower()
or p.lower() == "wananimate2pipeline"
p == "wananimate2distilledmodularpipeline"
or _is_wan_animate_2_distilled_path(p)
)
],
)
register_configs(
sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
pipeline_config_cls=Wan_Animate_2_14B_Config,
hf_model_paths=["Wan-AI/Wan2.2-Animate-2-14B-Diffusers"],
model_detectors=[
lambda p: (
p in ("wananimate2modularpipeline", "wananimate2pipeline")
or (
_is_wan_animate_2_path(p)
and not _is_wan_animate_2_distilled_path(p)
)
)
],
)
@@ -395,6 +395,27 @@ class Wan_Animate_2_14B_SamplingParam(Wan2_2_Base_SamplingParams):
)
@dataclass
class Wan_Animate_2_14B_Distilled_SamplingParam(Wan_Animate_2_14B_SamplingParam):
"""Sampling defaults for the distilled Wan-Animate-2 14B checkpoint (10 steps, no CFG)."""
# Guidance is distilled into the weights; the stage runs a single forward at 1.0.
guidance_scale: float = 1.0
num_inference_steps: int = 10
def _validate_with_pipeline_config(self, pipeline_config) -> None:
super()._validate_with_pipeline_config(pipeline_config)
# Runs once per request on the merged params. ValueError like the base hook's
# input checks: the /v1/videos handler maps it to HTTP 400.
if self.guidance_scale != 1.0:
raise ValueError(
f"guidance_scale={self.guidance_scale} is not supported by the "
"distilled Wan-Animate-2 checkpoint: it samples at guidance_scale 1.0 "
"only (the distilled recipe has no unconditional branch); omit the "
"field or set it to 1.0."
)
# Smallest clip: one conditioning frame plus one VAE temporal stride (4k+1, k >= 1).
MIN_CLIP_LEN = 5
+14 -1
View File
@@ -49,6 +49,7 @@ from sglang.multimodal_gen.runtime.utils.external_model_package import (
load_external_model_package,
)
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
load_dict,
maybe_download_model_index,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -473,10 +474,22 @@ def _get_config_info(
config = maybe_download_model_index(model_path)
pipeline_name = config.get("_class_name", "")
pipeline_name = pipeline_name.lower()
# A local checkpoint that ships modular_model_index.json next to model_index.json
# may name a finer-grained pipeline there (the Wan-Animate-2 variants);
# maybe_download_model_index reads the modular file only when model_index.json is
# absent, so probe the detectors with its class as well.
modular_pipeline_name = (
load_dict(os.path.join(model_path, "modular_model_index.json"))
.get("_class_name", "")
.lower()
)
probes = [model_path.lower(), pipeline_name]
if modular_pipeline_name and modular_pipeline_name != pipeline_name:
probes.append(modular_pipeline_name)
matched_model_names = []
for model_id, detector in _MODEL_NAME_DETECTORS:
if detector(model_path.lower()) or detector(pipeline_name):
if any(detector(probe) for probe in probes):
logger.debug(
f"Matched model name '{model_id}' using a registered detector."
)
@@ -9,11 +9,20 @@ standard component loaders. Text uses TextEncodingStage; CLIP and VAE adapters
preserve the reference preprocessing and latent scaling.
"""
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
Wan_Animate_2_14B_Distilled_Config,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
SchedulerLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
TransformerLoader,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
DPMSolverMultistepScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
@@ -25,6 +34,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.w
WanAnimate2VaeAdapter,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
class WanAnimate2TransformerLoader(TransformerLoader):
@@ -34,20 +46,51 @@ class WanAnimate2TransformerLoader(TransformerLoader):
strict_checkpoint_keys = True
class WanAnimate2SchedulerLoader(SchedulerLoader):
"""Builds DPM-Solver++ 2M from the flow-matching settings for the distilled checkpoint
and defers to the shared loader (the checkpoint's own scheduler/) for the base one;
the cookbook page explains why the shipped scheduler config is not read."""
# Pipeline-local: an empty component_names keeps this subclass out of the shared
# loader registry (ComponentLoader.__init_subclass__ registers by component name).
component_names = []
def load_customized(
self, component_model_path: str, server_args: ServerArgs, *args
):
pipeline_config = server_args.pipeline_config
if isinstance(pipeline_config, Wan_Animate_2_14B_Distilled_Config):
override = pipeline_config.scheduler_class_override
if override:
logger.warning(
"--scheduler-class-override %s is ignored for the distilled "
"Wan-Animate-2 checkpoint, which samples with DPM-Solver++ 2M on "
"the official sigma grid; drop the flag.",
override,
)
return DPMSolverMultistepScheduler(
num_train_timesteps=1000,
prediction_type="flow_prediction",
use_flow_sigmas=True,
flow_shift=pipeline_config.flow_shift,
)
return super().load_customized(component_model_path, server_args, *args)
class WanAnimate2Pipeline(ComposedPipelineBase):
"""Wan-Animate-2 14B in-context reference image + reference video -> video pipeline."""
pipeline_name = "WanAnimate2Pipeline"
is_video_pipeline = True
# All seven model_index.json components. The scheduler is built from the
# checkpoint's scheduler/scheduler_config.json (DPM-Solver++, flow prediction,
# flow sigmas) by the shared loader. Two of its settings are overridden at
# runtime: the stages call set_timesteps(sigmas=...) with the official Wan
# sigma grid at every clip (sigma_0 is exactly 1.0), shifted by
# pipeline_config.flow_shift, so the JSON's flow_shift and timestep_spacing
# are not consulted. The JSON's beta_schedule only shapes the beta-derived
# tables, which the flow-sigma path does not read.
# All seven model_index.json components; the scheduler comes from
# WanAnimate2SchedulerLoader (DPM-Solver++, flow prediction, flow sigmas for
# both checkpoints). Two of its settings are overridden at runtime: the
# stages call set_timesteps(sigmas=...) with the official Wan sigma grid at
# every clip (sigma_0 is exactly 1.0), shifted by pipeline_config.flow_shift,
# so the JSON's flow_shift and timestep_spacing are not consulted. The JSON's
# beta_schedule only shapes the beta-derived tables, which the flow-sigma
# path does not read.
_required_config_modules: list[str] = [
"text_encoder",
"tokenizer",
@@ -57,7 +100,10 @@ class WanAnimate2Pipeline(ComposedPipelineBase):
"image_processor",
"scheduler",
]
component_loaders = {"transformer": WanAnimate2TransformerLoader}
component_loaders = {
"transformer": WanAnimate2TransformerLoader,
"scheduler": WanAnimate2SchedulerLoader,
}
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
# Keep native modules visible to residency and layerwise-offload management;
@@ -39,6 +39,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
SANA_WM_TI2V_CI_sampling_params,
T2I_sampling_params,
T2V_sampling_params,
WAN_2_2_ANIMATE_2_14B_DISTILLED_sampling_params,
WAN_2_2_ANIMATE_2_14B_sampling_params,
_make_modelopt_ci_case,
_with_default_num_gpus,
@@ -64,6 +65,7 @@ from sglang.multimodal_gen.test.test_utils import (
DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_ANIMATE_2_14B_DISTILLED_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
@@ -478,6 +480,21 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
run_perf_check=True,
run_component_accuracy_check=False,
),
# Distilled checkpoint on the same inputs and server layout: 10 steps at guidance
# 1.0, so the denoiser runs the single-forward path (no unconditional batch).
DiffusionTestCase(
"wan2_2_animate_2_14b_distilled_ref2v",
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_DISTILLED_MODEL_NAME_FOR_TEST,
modality="video",
dit_layerwise_offload=True,
text_encoder_cpu_offload=True,
extras=["--layerwise-offload-components", "transformer"],
),
WAN_2_2_ANIMATE_2_14B_DISTILLED_sampling_params,
run_perf_check=True,
run_component_accuracy_check=False,
),
# flaky
# === Helios T2V ===
# DiffusionTestCase(
@@ -3561,6 +3561,35 @@
"estimated_full_test_time_s": 489.9,
"warmup_peak_vram_mb": 14598.0
},
"wan2_2_animate_2_14b_distilled_ref2v": {
"stages_ms": {
"WanAnimate2BeforeDenoisingStage": 7846.04,
"WanAnimate2DenoisingStage": 69260.73,
"WanAnimate2OutputStage": 0.08
},
"denoise_step_ms": {
"0": 23165.59,
"1": 3081.14,
"2": 4366.18,
"3": 4368.35,
"4": 4373.6,
"5": 4377.11,
"6": 4376.38,
"7": 4380.89,
"8": 4382.1,
"9": 4374.26
},
"expected_e2e_ms": 76924.33,
"expected_load_ms": 48892.73,
"expected_avg_denoise_ms": 6117.67,
"expected_median_denoise_ms": 4375.32,
"load_peak_vram_mb": 1776.0,
"runtime_peak_vram_mb": 36322.0,
"load_peak_allocated_mb": 1737.0,
"runtime_peak_allocated_mb": 25748.0,
"estimated_full_test_time_s": 439.9,
"warmup_peak_vram_mb": 14598.0
},
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
"stages_ms": {
"WanAnimate2BeforeDenoisingStage": 5778.62,
@@ -939,6 +939,17 @@ WAN_2_2_ANIMATE_2_14B_sampling_params = DiffusionSamplingParams(
},
)
# Distilled checkpoint: guidance is folded into the weights, so the request runs the
# model's 10-step, guidance-1.0 recipe over the same inputs, canvas and clip length.
WAN_2_2_ANIMATE_2_14B_DISTILLED_sampling_params = replace(
WAN_2_2_ANIMATE_2_14B_sampling_params,
extras={
**WAN_2_2_ANIMATE_2_14B_sampling_params.extras,
"num_inference_steps": 10,
"guidance_scale": 1.0,
},
)
HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams(
prompt="",
image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png",
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
# publish.
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
SGL_TEST_FILES_CI_DATA_REVISION = "75992dbf1d6eaeacbc1c2cd6beb48f77dd7ae4f7"
SGL_TEST_FILES_CI_DATA_REVISION = "252df4958ae6cc9f6b146b2f70382f75459235ad"
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
# when it's regenerated on its own cadence. 2173af2f re-generated the MiniMax-H3
@@ -208,6 +208,9 @@ DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-I2V-A14B-Diffusers
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST = (
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
)
DEFAULT_WAN_2_2_ANIMATE_2_14B_DISTILLED_MODEL_NAME_FOR_TEST = (
"Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"
)
# MOVA video generation models
DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST = "OpenMOSS-Team/MOVA-360p"
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import patch
import pytest
@@ -78,3 +80,58 @@ def default_global_server_args():
yield
finally:
set_global_server_args(previous)
@contextmanager
def mock_cuda_platform(
*,
memory_gb: int = 80,
available_memory_gb: int | dict[int, int] | None = None,
):
"""current_platform as an idle CUDA device of memory_gb GB, so ServerArgs resolution
and parallelism auto-tuning run without a GPU."""
def get_available_gpu_memory(device_id=0, **_kwargs):
if isinstance(available_memory_gb, dict):
return available_memory_gb[device_id]
if available_memory_gb is not None:
return available_memory_gb
return memory_gb
with (
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
return_value=False,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_mps",
return_value=False,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
return_value=True,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_rocm",
return_value=False,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_total_memory",
return_value=memory_gb * 1024**3,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.get_available_gpu_memory",
side_effect=get_available_gpu_memory,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
return_value=True,
),
):
yield
@pytest.fixture
def cuda_platform():
with mock_cuda_platform():
yield
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
"""Model routing in the registry, without Hub access."""
import json
import pytest
from sglang.multimodal_gen import registry
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
Wan_Animate_2_14B_Config,
Wan_Animate_2_14B_Distilled_Config,
)
@pytest.fixture(autouse=True)
def clear_registry_cache():
registry.get_model_info.cache_clear()
registry._get_config_info.cache_clear()
yield
registry.get_model_info.cache_clear()
registry._get_config_info.cache_clear()
def _checkpoint_dir(
root,
*,
name: str = "ckpt-7f3a",
index_class: str = "SomePipeline",
modular_class: str | None = None,
) -> str:
"""Laid out like a published repo: model_index.json, the component folders the index
verifier looks for, and modular_model_index.json when a class is given."""
ckpt = root / name
ckpt.mkdir(parents=True)
(ckpt / "transformer").mkdir()
(ckpt / "vae").mkdir()
index = {"_class_name": index_class, "_diffusers_version": "0.40.0.dev0"}
(ckpt / "model_index.json").write_text(json.dumps(index))
if modular_class is not None:
(ckpt / "modular_model_index.json").write_text(
json.dumps({**index, "_class_name": modular_class})
)
return str(ckpt)
@pytest.mark.parametrize("modular_class", [None, "SomeModularPipeline"])
def test_detectors_are_probed_with_the_modular_pipeline_class(
tmp_path, monkeypatch, modular_class
):
# A checkpoint may ship modular_model_index.json next to model_index.json and name
# a finer-grained pipeline there; without it the probes are the path and the
# model_index class only.
probes = []
def detector(probe: str) -> bool:
probes.append(probe)
return False
monkeypatch.setattr(registry, "_MODEL_NAME_DETECTORS", [("probe", detector)])
path = _checkpoint_dir(tmp_path, modular_class=modular_class)
assert registry._get_config_info(path) is None
expected = [path.lower(), "somepipeline"]
if modular_class is not None:
expected.append(modular_class.lower())
assert probes == expected
# Wan-Animate-2: both repos ship model_index.json with the same _class_name;
# modular_model_index.json names the variant, and a distill token in the directory's
# own name selects the distilled entry.
_WAN_ANIMATE_2 = "WanAnimate2Pipeline"
@pytest.mark.parametrize(
"modular_class, config_cls",
[
("WanAnimate2ModularPipeline", Wan_Animate_2_14B_Config),
("WanAnimate2DistilledModularPipeline", Wan_Animate_2_14B_Distilled_Config),
# A mirror that skipped modular_model_index.json leaves only the shared class.
(None, Wan_Animate_2_14B_Config),
],
)
def test_neutral_directory_routes_by_its_modular_pipeline_class(
tmp_path, modular_class, config_cls
):
path = _checkpoint_dir(
tmp_path, index_class=_WAN_ANIMATE_2, modular_class=modular_class
)
assert registry._get_config_info(path).pipeline_config_cls is config_cls
def test_distill_token_in_the_directory_name_routes_to_the_distilled_config(tmp_path):
# The token routes without the modular index; the shared model_index class matches
# the base detector too, so the distilled entry is registered first.
path = _checkpoint_dir(
tmp_path, name="wan_animate_2_distilled", index_class=_WAN_ANIMATE_2
)
info = registry._get_config_info(path)
assert info.pipeline_config_cls is Wan_Animate_2_14B_Distilled_Config
def test_distill_token_in_a_parent_directory_leaves_the_checkpoint_in_charge(tmp_path):
# Only the directory's own name counts: a base checkpoint kept under a parent such
# as distillation-runs/ routes by its modular class.
path = _checkpoint_dir(
tmp_path / "distillation-runs",
name="wan_animate_2_base",
index_class=_WAN_ANIMATE_2,
modular_class="WanAnimate2ModularPipeline",
)
assert (
registry._get_config_info(path).pipeline_config_cls is Wan_Animate_2_14B_Config
)
def test_model_id_short_name_selects_the_variant_for_a_neutral_directory():
info = registry._get_config_info(
"/models/ckpt-7f3a", model_id="Wan2.2-Animate-2-14B-Distilled-Diffusers"
)
assert info.pipeline_config_cls is Wan_Animate_2_14B_Distilled_Config
@@ -4,7 +4,7 @@ import subprocess
import sys
import tempfile
import unittest
from contextlib import ExitStack, contextmanager
from contextlib import ExitStack
from unittest.mock import Mock, patch
import zmq
@@ -97,52 +97,7 @@ from sglang.multimodal_gen.runtime.server_args import (
)
from sglang.multimodal_gen.runtime.server_args import server_args as server_args_module
from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser
@contextmanager
def _mock_cuda_platform(
*,
memory_gb: int = 80,
available_memory_gb: int | dict[int, int] | None = None,
):
def get_available_gpu_memory(device_id=0, **_kwargs):
if isinstance(available_memory_gb, dict):
return available_memory_gb[device_id]
if available_memory_gb is not None:
return available_memory_gb
return memory_gb
with (
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cpu",
return_value=False,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_mps",
return_value=False,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_cuda",
return_value=True,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_rocm",
return_value=False,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.get_device_total_memory",
return_value=memory_gb * 1024**3,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.get_available_gpu_memory",
side_effect=get_available_gpu_memory,
),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.enable_dit_layerwise_offload_by_default",
return_value=True,
),
):
yield
from sglang.multimodal_gen.test.unit.conftest import mock_cuda_platform
def _from_dict_without_model_resolution(
@@ -151,7 +106,7 @@ def _from_dict_without_model_resolution(
pipeline_config = pipeline_config or QwenImagePipelineConfig()
with (
patch.object(PipelineConfig, "from_kwargs", return_value=pipeline_config),
_mock_cuda_platform(),
mock_cuda_platform(),
):
return ServerArgs.from_dict(kwargs)
@@ -191,7 +146,7 @@ class _CudaPlatformTestCase(unittest.TestCase):
# Residency queries inspect the platform after argument construction too.
stack = ExitStack()
self.addCleanup(stack.close)
stack.enter_context(_mock_cuda_platform())
stack.enter_context(mock_cuda_platform())
class TestSchedulerEndpoints(unittest.TestCase):
@@ -284,7 +239,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
patch.object(
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
),
_mock_cuda_platform(),
mock_cuda_platform(),
):
args, unknown_args = parser.parse_known_args(argv)
server_args = ServerArgs.from_cli_args(args, unknown_args)
@@ -570,7 +525,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
patch.object(
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
),
_mock_cuda_platform(),
mock_cuda_platform(),
):
args, unknown_args = parser.parse_known_args(argv)
server_args = ServerArgs.from_cli_args(args, unknown_args)
@@ -591,7 +546,7 @@ class TestServerArgsPathExpansion(_CudaPlatformTestCase):
patch.object(
PipelineConfig, "from_kwargs", return_value=QwenImagePipelineConfig()
),
_mock_cuda_platform(),
mock_cuda_platform(),
):
args, unknown_args = parser.parse_known_args(argv)
server_args = ServerArgs.from_cli_args(args, unknown_args)
@@ -3550,7 +3505,7 @@ class TestPipelineResolutionCliOverride(unittest.TestCase):
with (
patch.object(sys, "argv", ["sglang"] + argv),
_mock_cuda_platform(),
mock_cuda_platform(),
):
args, unknown_args = parser.parse_known_args(argv)
server_args = ServerArgs.from_cli_args(args, unknown_args)
@@ -3569,7 +3524,7 @@ class TestPipelineResolutionCliOverride(unittest.TestCase):
with (
patch.object(sys, "argv", ["sglang"] + argv),
_mock_cuda_platform(),
mock_cuda_platform(),
):
args, unknown_args = parser.parse_known_args(argv)
server_args = ServerArgs.from_cli_args(args, unknown_args)
@@ -23,12 +23,19 @@ from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
WanAnimate2ArchConfig,
WanAnimate2Config,
)
from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan_Animate_2_14B_Config
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
Wan_Animate_2_14B_Config,
Wan_Animate_2_14B_Distilled_Config,
)
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
)
from sglang.multimodal_gen.configs.sample.wan import Wan_Animate_2_14B_SamplingParam
from sglang.multimodal_gen.configs.sample.wan import (
Wan_Animate_2_14B_Distilled_SamplingParam,
Wan_Animate_2_14B_SamplingParam,
)
from sglang.multimodal_gen.registry import _get_config_info
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
@@ -46,6 +53,7 @@ from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend i
)
from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
SchedulerLoader,
_supported_init_kwargs,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
@@ -67,6 +75,7 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multi
)
from sglang.multimodal_gen.runtime.pipelines.wan_animate_2_pipeline import (
WanAnimate2Pipeline,
WanAnimate2SchedulerLoader,
WanAnimate2TransformerLoader,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
@@ -117,8 +126,12 @@ def test_pipeline_local_loader_preserves_shared_transformer_registry():
assert component_name_to_loader_cls[name] is TransformerLoader
assert not TransformerLoader.strict_checkpoint_keys
assert WanAnimate2TransformerLoader.strict_checkpoint_keys
for name in SchedulerLoader.component_names:
assert component_name_to_loader_cls[name] is SchedulerLoader
assert WanAnimate2SchedulerLoader.component_names == []
assert WanAnimate2Pipeline.component_loaders == {
"transformer": WanAnimate2TransformerLoader
"transformer": WanAnimate2TransformerLoader,
"scheduler": WanAnimate2SchedulerLoader,
}
@@ -241,6 +254,57 @@ def _parallel_server_args(
)
def _two_gpu_server_args(pipeline_config, **kwargs) -> ServerArgs:
"""ServerArgs for the config's HF id with the config pinned and model_index.json
served from memory; under the cuda_platform fixture _adjust_parallelism runs
without a GPU or network."""
hf_id = "Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
if isinstance(pipeline_config, Wan_Animate_2_14B_Distilled_Config):
hf_id = "Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers"
with (
patch.object(PipelineConfig, "from_kwargs", return_value=pipeline_config),
patch(
"sglang.multimodal_gen.registry.maybe_download_model_index",
return_value={"_class_name": "WanAnimate2Pipeline"},
),
patch(
"sglang.multimodal_gen.runtime.server_args.auto_tune.dit_parameter_count",
return_value=None,
),
):
return ServerArgs.from_dict({"model_path": hf_id, "num_gpus": 2, **kwargs})
def test_two_gpu_default_layout_splits_cfg_only_for_the_base_checkpoint(
cuda_platform,
):
# With no parallelism flags the server auto-enables CFG parallel for a model whose
# defaults use guidance; the distilled checkpoint runs one branch, so its second GPU
# goes to sequence parallelism instead.
base = _two_gpu_server_args(Wan_Animate_2_14B_Config())
distilled = _two_gpu_server_args(Wan_Animate_2_14B_Distilled_Config())
assert base.enable_cfg_parallel and not distilled.enable_cfg_parallel
assert (base.cfg_parallel_degree, base.sp_degree) == (2, 1)
assert (distilled.cfg_parallel_degree, distilled.sp_degree) == (1, 2)
def test_distilled_rejects_explicit_cfg_parallel(cuda_platform):
# An explicit CFG group would silently run the single branch on every rank.
with pytest.raises(ValueError, match="does not support CFG parallelism"):
_two_gpu_server_args(
Wan_Animate_2_14B_Distilled_Config(), enable_cfg_parallel=True
)
def test_distilled_deployment_config_keeps_the_base_residency_policy():
# Only the CFG-parallel field differs; a change to the base residency policy must
# reach the distilled variant.
base = Wan_Animate_2_14B_Config().get_model_deployment_config()
distilled = Wan_Animate_2_14B_Distilled_Config().get_model_deployment_config()
assert distilled.supports_cfg_parallel is False
assert replace(distilled, supports_cfg_parallel=True) == base
def test_validate_server_args_rejects_fsdp_before_any_weights_load():
with pytest.raises(NotImplementedError, match="use-fsdp-inference"):
Wan_Animate_2_14B_Config().validate_server_args(
@@ -487,6 +551,16 @@ def test_checkpoint_config_without_image_embedding_is_rejected():
config.update_model_arch({"use_img_emb": False})
def test_distilled_log_scale_defaults_to_the_official_value_unless_the_checkpoint_sets_it():
# -1.3 is the official wan_animate_2_distillation.yaml value; the published
# transformer/config.json has no log_scale key, so this default is what runs, and a
# checkpoint that does carry the key wins over it.
config = Wan_Animate_2_14B_Distilled_Config().dit_config
assert config.log_scale == -1.3
config.update_model_arch({"log_scale": -0.7})
assert config.log_scale == -0.7
# Wan_Animate_2_14B_SamplingParam
@@ -580,20 +654,91 @@ def test_sampling_params_reject_the_shared_dit_cache_heuristics(field_name):
Wan_Animate_2_14B_SamplingParam(**{field_name: True})
def _distilled_request(**user_kwargs) -> SamplingParams:
"""A /v1/videos-style request through the public entry point: registry defaults for
the distilled HF id merged with the user fields, adjusted and validated."""
server_args = SimpleNamespace(
pipeline_class_name=None,
backend="sglang",
model_id=None,
pipeline_config=Wan_Animate_2_14B_Distilled_Config(),
output_path=None,
num_gpus=1,
comfyui_mode=False,
)
with patch(
"sglang.multimodal_gen.registry.maybe_download_model_index",
return_value={"_class_name": "WanAnimate2Pipeline"},
):
return SamplingParams.from_user_sampling_params_args(
"Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
server_args,
prompt="a person",
image_path="reference.png",
video_path="driving.mp4",
**user_kwargs,
)
@pytest.mark.parametrize(
"guidance_scale, rejected",
[(3.0, True), (0.5, True), (1.0, False), (None, False)],
)
def test_distilled_guidance_other_than_one_is_rejected(guidance_scale, rejected):
# The distilled recipe has no unconditional branch: the once-per-request validation
# hook raises ValueError (HTTP 400 on /v1/videos) for every guidance_scale but 1.0;
# an omitted field (None) takes the class default.
user_kwargs = {} if guidance_scale is None else {"guidance_scale": guidance_scale}
if rejected:
with pytest.raises(
ValueError,
match=rf"guidance_scale={guidance_scale} .*guidance_scale 1\.0 only",
):
_distilled_request(**user_kwargs)
return
params = _distilled_request(**user_kwargs)
assert params.guidance_scale == 1.0
def test_distilled_sampling_defaults_run_without_cfg():
# The official distilled recipe: 10 steps, guidance 1.0; the rest is inherited.
params = Wan_Animate_2_14B_Distilled_SamplingParam()
assert params.guidance_scale == 1.0
assert params.num_inference_steps == 10
# Registry resolution (no network)
def test_registered_hf_path_resolves_to_wan_animate_configs():
info = _get_config_info("Wan-AI/Wan2.2-Animate-2-14B-Diffusers")
@pytest.mark.parametrize(
"hf_id, pipeline_cls, sampling_cls",
[
(
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
Wan_Animate_2_14B_Config,
Wan_Animate_2_14B_SamplingParam,
),
(
"Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers",
Wan_Animate_2_14B_Distilled_Config,
Wan_Animate_2_14B_Distilled_SamplingParam,
),
],
)
def test_registered_hf_path_resolves_to_wan_animate_configs(
hf_id, pipeline_cls, sampling_cls
):
info = _get_config_info(hf_id)
assert info is not None
assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
assert info.sampling_param_cls is Wan_Animate_2_14B_SamplingParam
assert info.pipeline_config_cls is pipeline_cls
assert info.sampling_param_cls is sampling_cls
@pytest.mark.parametrize(
"path", ["some/local/dir/wan_animate_2", "my-wan2.2-animate-2-ckpt"]
)
def test_detector_resolves_wan_animate_style_paths(path):
# No variant token in the name: the base classes, not the distilled ones.
with patch(
"sglang.multimodal_gen.registry.maybe_download_model_index",
return_value={},
@@ -606,7 +751,8 @@ def test_detector_resolves_wan_animate_style_paths(path):
def test_model_index_class_name_routes_a_checkpoint_at_any_path():
# A local checkpoint directory need not carry a model token in its name; its
# model_index.json _class_name is what identifies it.
# model_index.json _class_name is what identifies it. Both variants share that
# name; modular_model_index.json tells them apart (below).
with patch(
"sglang.multimodal_gen.registry.maybe_download_model_index",
return_value={"_class_name": "WanAnimate2Pipeline"},
@@ -616,15 +762,22 @@ def test_model_index_class_name_routes_a_checkpoint_at_any_path():
assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
def test_v1_animate_checkpoint_does_not_route_to_wan_animate_2():
@pytest.mark.parametrize(
"path", ["Wan-AI/Wan2.2-Animate-14B", "ckpts/Wan2.2-Animate-14B-distill"]
)
def test_v1_animate_checkpoint_does_not_route_to_wan_animate_2(path):
# Wan2.2-Animate-14B (v1) is a different model and pipeline; a detector loosened to
# "animate" would pass the positive cases above and mis-route it here.
# "animate" would pass the positive cases above and mis-route it here, and the
# distilled detector needs the "-2-"/"_2" token as well as "distill".
with patch(
"sglang.multimodal_gen.registry.maybe_download_model_index",
return_value={"_class_name": "WanAnimatePipeline"},
):
info = _get_config_info("Wan-AI/Wan2.2-Animate-14B")
assert info is None or info.pipeline_config_cls is not Wan_Animate_2_14B_Config
info = _get_config_info(path)
assert info is None or info.pipeline_config_cls not in (
Wan_Animate_2_14B_Config,
Wan_Animate_2_14B_Distilled_Config,
)
# /v1/videos routing: the model-specific knobs are declared extras, the reference image and
@@ -1305,8 +1458,10 @@ def test_extract_inputs_rejects_batched_single_value_fields(overrides, message):
_WanAnimate2Inputs.from_request(batch)
# Scheduler: loaded from the checkpoint's scheduler/scheduler_config.json through the shared
# SchedulerLoader like the other components. The stages pass the official sigma grid to
# Scheduler: a loaded component like the others, through WanAnimate2SchedulerLoader. The
# base checkpoint's scheduler/scheduler_config.json is loaded as is; for the distilled
# config the loader builds DPMSolverMultistepScheduler from the flow-matching settings
# instead of from its Euler scheduler_config.json. The stages pass the official sigma grid to
# set_timesteps explicitly, so the loaded configuration must denoise exactly like a
# DPMSolverMultistepScheduler built from the flow-matching settings alone.
@@ -1355,6 +1510,67 @@ _FLOW_MATCHING_SCHEDULER_KWARGS = {
}
def _load_scheduler(pipeline_config, checkpoint_config: dict | None):
"""Run WanAnimate2SchedulerLoader with the checkpoint's scheduler_config.json served
from memory (None: reading it is an error) and the flag-free server args it reads."""
reader = {"return_value": dict(checkpoint_config)} if checkpoint_config else {}
server_args = SimpleNamespace(pipeline_config=pipeline_config)
with patch(
"sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader"
".get_diffusers_component_config",
**reader,
) as read:
scheduler = WanAnimate2SchedulerLoader().load_customized(
"/ckpt/scheduler", server_args
)
assert read.called is (checkpoint_config is not None)
return scheduler
def test_base_config_loads_the_checkpoint_scheduler_as_is():
# The shared loader path: the class the checkpoint names, built from its own settings
# (beta_schedule=linear comes only from that file).
scheduler = _load_scheduler(
Wan_Animate_2_14B_Config(),
{"_class_name": "DPMSolverMultistepScheduler", **_CHECKPOINT_SCHEDULER_CONFIG},
)
assert isinstance(scheduler, DPMSolverMultistepScheduler)
assert scheduler._inner.config.beta_schedule == "linear"
assert scheduler._inner.config.flow_shift == 5.0
def test_distilled_config_gets_dpm_solver_built_from_the_flow_matching_settings():
# The wrapper comes from the flow-matching settings and the configured flow_shift;
# the shipped scheduler_config.json, which carries none of them, is never read.
config = Wan_Animate_2_14B_Distilled_Config()
config.update_config_from_dict({"flow_shift": 3.0})
scheduler = _load_scheduler(config, None)
assert isinstance(scheduler, DPMSolverMultistepScheduler)
inner = scheduler._inner.config
assert inner.algorithm_type == "dpmsolver++"
assert inner.solver_order == 2
for key, value in {**_FLOW_MATCHING_SCHEDULER_KWARGS, "flow_shift": 3.0}.items():
assert getattr(inner, key) == value, key
def test_distilled_config_ignores_the_scheduler_class_override_with_one_warning(caplog):
# The flag would select the shipped Euler scheduler on the base path; the distilled
# path keeps DPM-Solver++ and says so instead of silently ignoring the flag.
config = Wan_Animate_2_14B_Distilled_Config()
config.scheduler_class_override = "FlowMatchEulerDiscreteScheduler"
with caplog.at_level(logging.WARNING):
scheduler = _load_scheduler(config, None)
assert isinstance(scheduler, DPMSolverMultistepScheduler)
inner = scheduler._inner.config
assert inner.algorithm_type == "dpmsolver++"
for key, value in _FLOW_MATCHING_SCHEDULER_KWARGS.items():
assert getattr(inner, key) == value, key
warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
assert len(warnings) == 1, warnings
assert "--scheduler-class-override FlowMatchEulerDiscreteScheduler" in warnings[0]
assert "ignored for the distilled Wan-Animate-2 checkpoint" in warnings[0]
def test_loader_keeps_the_checkpoint_scheduler_settings():
kwargs = _supported_init_kwargs(
DPMSolverMultistepScheduler, dict(_CHECKPOINT_SCHEDULER_CONFIG)