[diffusion] model: support Wan-Animate-2 (#42941)

Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Ankit Dhall
2026-10-09 20:46:23 +08:00
committed by GitHub
co-authored by Ankit Dhall Mick Qian
parent 0955169ed2
commit f1e0472fcc
32 changed files with 6815 additions and 116 deletions
@@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.models.dits.sana_video import SanaVideoConfig
from sglang.multimodal_gen.configs.models.dits.stablediffusion3 import (
StableDiffusion3TransformerConfig,
)
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig
__all__ = [
@@ -34,6 +35,7 @@ __all__ = [
"LingBotVideoMoEConfig",
"LongLive2VideoConfig",
"MiniMaxH3DiTConfig",
"WanAnimate2Config",
"WanVideoConfig",
"Hunyuan3DDiTConfig",
"MOVAAudioConfig",
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
#
# Wan-Animate-2 DiT config. The checkpoint is the Wan2.2-I2V-14B backbone with no
# extra parameter tensors, so WanVideoArchConfig is reused with in_channels=36,
# image_dim=1280, added_kv_proj_dim=5120 and a param-name remap.
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.base import DiTConfig
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoArchConfig
# Checkpoint block params use the official Wan names nested under `blocks.{i}.block.*`.
# The loader applies mapping rules in insertion order, re-scanning after each hit, so the
# extras below chain into the reused Wan lora (official -> hf) and main (hf -> native) maps.
_WAN = WanVideoArchConfig()
_WAN_ANIMATE_2_EXTRA_MAPPING: dict[str, str] = {
# Strip the `.block` infix so the reused Wan rules match.
r"^blocks\.(\d+)\.block\.(.*)$": r"blocks.\1.\2",
# Self-attn / cross-attn qk-norms (not covered by the Wan lora map).
r"^blocks\.(\d+)\.self_attn\.norm_q\.(.*)$": r"blocks.\1.norm_q.\2",
r"^blocks\.(\d+)\.self_attn\.norm_k\.(.*)$": r"blocks.\1.norm_k.\2",
r"^blocks\.(\d+)\.cross_attn\.norm_q\.(.*)$": r"blocks.\1.attn2.norm_q.\2",
r"^blocks\.(\d+)\.cross_attn\.norm_k\.(.*)$": r"blocks.\1.attn2.norm_k.\2",
# Cross-attn I2V image branch -> WanI2VCrossAttention add_k_proj/add_v_proj/norm_added_k.
r"^blocks\.(\d+)\.cross_attn\.k_img\.(.*)$": r"blocks.\1.attn2.add_k_proj.\2",
r"^blocks\.(\d+)\.cross_attn\.v_img\.(.*)$": r"blocks.\1.attn2.add_v_proj.\2",
r"^blocks\.(\d+)\.cross_attn\.norm_k_img\.(.*)$": r"blocks.\1.attn2.norm_added_k.\2",
# The Diffusers checkpoint keeps the official self_attn/cross_attn prefixes but names
# the projections the diffusers way (to_q/to_k/to_v/to_out.0, add_*_proj).
r"^blocks\.(\d+)\.self_attn\.to_out\.0\.(.*)$": r"blocks.\1.to_out.\2",
r"^blocks\.(\d+)\.self_attn\.(to_q|to_k|to_v)\.(.*)$": r"blocks.\1.\2.\3",
r"^blocks\.(\d+)\.cross_attn\.to_out\.0\.(.*)$": r"blocks.\1.attn2.to_out.\2",
r"^blocks\.(\d+)\.cross_attn\.(to_q|to_k|to_v|add_k_proj|add_v_proj|norm_added_k)\.(.*)$": r"blocks.\1.attn2.\2.\3",
# `norm3` is the block's only affine LayerNorm and feeds cross-attn; the reused rules
# only know its hf name (`norm2`).
r"^blocks\.(\d+)\.norm3\.(.*)$": r"blocks.\1.self_attn_residual_norm.norm.\2",
# Per-block 6-way modulation -> scale_shift_table.
r"^blocks\.(\d+)\.modulation$": r"blocks.\1.scale_shift_table",
# Output head (`head.norm` is param-free; no-op if absent).
r"^head\.head\.(.*)$": r"proj_out.\1",
r"^head\.modulation$": r"scale_shift_table",
r"^head\.norm\.(.*)$": r"norm_out.\1",
# Embedders in official naming; hf-named checkpoints are covered by the reused rules.
r"^text_embedding\.0\.(.*)$": r"condition_embedder.text_embedder.fc_in.\1",
r"^text_embedding\.2\.(.*)$": r"condition_embedder.text_embedder.fc_out.\1",
r"^time_embedding\.0\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_in.\1",
r"^time_embedding\.2\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_out.\1",
r"^time_projection\.1\.(.*)$": r"condition_embedder.time_modulation.linear.\1",
r"^img_emb\.proj\.0\.(.*)$": r"condition_embedder.image_embedder.norm1.\1",
r"^img_emb\.proj\.1\.(.*)$": r"condition_embedder.image_embedder.ff.fc_in.\1",
r"^img_emb\.proj\.3\.(.*)$": r"condition_embedder.image_embedder.ff.fc_out.\1",
r"^img_emb\.proj\.4\.(.*)$": r"condition_embedder.image_embedder.norm2.\1",
# patch_embedding, self/cross-attn q/k/v/o and ffn.* are covered by the reused Wan maps.
}
@dataclass
class WanAnimate2ArchConfig(WanVideoArchConfig):
"""Wan2.2-I2V-14B backbone dims (inherited) with the Wan-Animate-2 I/O + remap."""
# Extras first, then Wan lora (official -> hf), then Wan main (hf -> native).
param_names_mapping: dict[str, str] = field(
default_factory=lambda: {
**_WAN_ANIMATE_2_EXTRA_MAPPING,
**_WAN.lora_param_names_mapping,
**_WAN.param_names_mapping,
}
)
# Only used for re-serialization, which the inference path does not exercise.
reverse_param_names_mapping: dict[str, str] = field(
default_factory=lambda: dict(_WAN.reverse_param_names_mapping)
)
# LoRA adapters use the official Wan names; the loader applies this before
# param_names_mapping.
lora_param_names_mapping: dict[str, str] = field(
default_factory=lambda: dict(_WAN.lora_param_names_mapping)
)
# in_channels: 16 noise + 20 conditioning (4-ch i2v mask + 16-ch cond latent).
# image_dim: CLIP (XLM-RoBERTa ViT-H/14) embed dim -> image_embedder.
# added_kv_proj_dim -> WanI2VCrossAttention (add_k_proj/add_v_proj/norm_added_k).
in_channels: int = 36
out_channels: int = 16
image_dim: int | None = 1280
added_kv_proj_dim: int | None = 5120
num_layers: int = 40
# In-context mechanism (no extra params): reference-video RoPE offsets and the flex-attention
# score_mod, consumed by forward_ref / forward_gen. The refer_* / log_scale names
# match the official and diffusers transformer configs; do not rename. In those
# names "refer" means the reference video (the motion source), not the reference image.
refer_offset_t: int = 1
refer_offset_h: int = 0
refer_offset_w: int = (
-1
) # <0: resolved to the gen grid width in tokens at runtime (official convention)
refer_stride: int = 1
# Additive attention-score bias on the keys of generation frame 1 (official name);
# 0.0 disables it. Read from transformer/config.json when present; the base weights omit it.
log_scale: float = 0.0
# The image-embedding cross-attention (added_kv_proj_dim) is always built, as in the
# official model; the checkpoint config states the same and nothing else reads this.
use_img_emb: bool = True
def __post_init__(self) -> None:
# transformer/config.json uses the official Wan names for the backbone dims;
# update_model_arch parks unknown keys in extra_attrs, so translate them here.
extras = self.extra_attrs
if "num_heads" in extras:
self.num_attention_heads = extras.pop("num_heads")
if "dim" in extras:
self.attention_head_dim = extras.pop("dim") // self.num_attention_heads
if "in_dim" in extras:
self.in_channels = extras.pop("in_dim")
if "out_dim" in extras:
self.out_channels = extras.pop("out_dim")
if not self.use_img_emb:
raise ValueError(
"Wan-Animate-2 always builds the image-embedding cross-attention; "
"a checkpoint with use_img_emb=false is a different model."
)
self.patch_size = tuple(self.patch_size)
super().__post_init__()
@dataclass
class WanAnimate2Config(DiTConfig):
arch_config: WanAnimate2ArchConfig = field(default_factory=WanAnimate2ArchConfig)
prefix: str = "WanAnimate2"
@@ -3,11 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
import torch
from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig
from sglang.multimodal_gen.configs.models.dits import WanVideoConfig
from sglang.multimodal_gen.configs.models.dits import WanAnimate2Config, WanVideoConfig
from sglang.multimodal_gen.configs.models.encoders import (
BaseEncoderOutput,
CLIPVisionConfig,
@@ -45,6 +47,10 @@ def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tenso
return prompt_embeds_tensor
def animate2_postprocess_text(outputs: BaseEncoderOutput, text_inputs) -> torch.Tensor:
return outputs.last_hidden_state[0, : text_inputs["num_tokens"]]
@dataclass
class WanI2VCommonConfig(PipelineConfig):
# for all wan i2v pipelines
@@ -301,6 +307,127 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig):
)
@dataclass
class Wan_Animate_2_14B_Config(WanI2V720PConfig):
"""Wan-Animate-2 14B (Wan2.2): reference image + reference video -> video.
Shares the Wan I2V component bundle (T5, CLIP vision, Wan VAE with the encoder
loaded) and swaps in the in-context ``WanAnimate2Config`` DiT; a single expert, so no
``boundary_ratio`` switching. The bespoke stages read ``flow_shift`` from here
(carrying the audio track is a per-request sampling parameter); everything else below
restates the Wan2.2 14B defaults instead of inheriting the Wan2.1 ones from the base.
"""
# reference image + reference video -> video is I2V-shaped.
task_type: ModelTaskType = ModelTaskType.I2V
# In-context animation DiT (in_channels=36 = 16 noise + 20 conditioning).
dit_config: DiTConfig = field(default_factory=WanAnimate2Config)
# Denoising stage. Used when building the sigma grid.
flow_shift: float | None = 5.0
postprocess_text_funcs: tuple[Callable, ...] = field(
default_factory=lambda: (animate2_postprocess_text,)
)
def tokenize_prompt(self, prompt, tokenizer, tok_kwargs):
text_inputs = super().tokenize_prompt(
prompt,
tokenizer,
{**tok_kwargs, "max_length": self.dit_config.arch_config.text_len},
)
# Count on the host before TextEncodingStage uploads inputs; no GPU scalar sync.
text_inputs["num_tokens"] = int(text_inputs["attention_mask"][0].gt(0).sum())
return text_inputs
# The component manager casts the VAE to this dtype at its first use (the reference
# encode in the before-denoising stage), so it is the precision the whole bespoke
# VAE path runs at; fp32 like the other Wan2.2 configs changes the output.
vae_decode_precision: str = "bf16"
def get_model_deployment_config(self) -> ModelDeploymentConfig:
# The Wan2.2 A14B modes (auto mode may layerwise-offload the 14B DiT) with the
# resident threshold the base had: at or above 60 GB free the DiT stays resident.
return ModelDeploymentConfig(
dit_layerwise_offload_modes=("auto", "memory"),
auto_dit_offload_prefetch_size=2,
keep_resident_min_available_gb=60,
keep_resident_components=("dit",),
)
def get_decode_scale_and_shift(
self,
device: torch.device | None,
dtype: torch.dtype | None,
vae: torch.nn.Module | None,
) -> tuple[float, None]:
"""``(1.0, None)``: ``WanAnimate2VaeAdapter.decode`` applies the latent scaling itself,
so the DecodingStage must not de-normalize again."""
return 1.0, None
def supports_disaggregation(self) -> bool:
# The stages share one WanAnimate2RequestState object (GPU tensors, a torch.Generator,
# the clip schedule) through batch.extra. The disaggregation transport forwards only
# JSON scalars and tensor pytrees (dict/list/tuple) from batch.extra and never the
# generator (disaggregation/extra_tensors.py, scheduler_mixin._EXCLUDE_FIELDS), so a
# split-role deployment would silently drop the state and fail at the first request.
return False
def validate_server_args(self, server_args: Any) -> None:
if server_args.use_fsdp_inference:
raise NotImplementedError(
"Wan-Animate-2 does not support --use-fsdp-inference; use "
"--dit-layerwise-offload or --tp-size instead."
)
# Ring splits the K/V across ranks; the in-context block mask indexes the full
# [generation | reference-video] K/V on one rank.
if server_args.ring_degree > 1:
raise NotImplementedError(
"Wan-Animate-2 does not support ring sequence parallelism "
f"(--ring-degree {server_args.ring_degree}); use --ulysses-degree N "
"with --ring-degree 1."
)
num_heads = self.dit_config.num_attention_heads
if num_heads % server_args.tp_size != 0:
raise ValueError(
f"Wan-Animate-2 tensor parallelism needs the {num_heads} attention "
f"heads divisible by --tp-size {server_args.tp_size}."
)
local_num_heads = num_heads // server_args.tp_size
if local_num_heads % server_args.ulysses_degree != 0:
raise ValueError(
f"Wan-Animate-2 Ulysses SP needs the TP-local head count ({local_num_heads}) "
f"divisible by --ulysses-degree {server_args.ulysses_degree}."
)
# The in-context self-attention runs on torch flex_attention whatever backend is
# selected; only the cross-attention and the reference pass would follow the flag.
requested_dit_backend = server_args.requested_component_attention_backend(
"transformer"
)
if (
server_args.is_arg_explicitly_set("attention_backend")
and server_args.attention_backend is not None
) or requested_dit_backend is not None:
raise ValueError(
"Wan-Animate-2 does not support selecting the DiT attention backend "
f"(--attention-backend {server_args.attention_backend} / "
f"--component-attention-backends transformer={requested_dit_backend}): "
"its in-context self-attention runs on torch flex_attention only. Drop "
"the flag; the text encoder still takes "
"--component-attention-backends text_encoder=<backend>."
)
# The parent block applies the offline Q/K rotation inside its own attention
# path; forward_ref / forward_gen never do, so a rotated checkpoint would load
# and produce wrong output without a message.
if envs.SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION:
raise ValueError(
"Wan-Animate-2 does not support SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION "
"(offline Q/K rotation); unset it for this model."
)
super().validate_server_args(server_args)
# =============================================
# ============= Causal Self-Forcing =============
# =============================================
@@ -322,6 +449,7 @@ def register():
Wan2_2_I2V_A14B_SamplingParam,
Wan2_2_T2V_A14B_SamplingParam,
Wan2_2_TI2V_5B_SamplingParam,
Wan_Animate_2_14B_SamplingParam,
WanI2V_14B_480P_SamplingParam,
WanI2V_14B_720P_SamplingParam,
WanT2V_1_3B_SamplingParams,
@@ -423,3 +551,20 @@ def register():
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
],
)
register_configs(
sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
pipeline_config_cls=Wan_Animate_2_14B_Config,
hf_model_paths=[
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
],
model_detectors=[
# The registry calls this with the path and with model_index's lower-cased
# _class_name. Require the "-2-"/"_2" token so a v1 Wan2.2-Animate-14B
# checkpoint (a different model / pipeline) does not mis-route here.
lambda p: (
"wan_animate_2" in p.lower()
or "wan2.2-animate-2" in p.lower()
or p.lower() == "wananimate2pipeline"
)
],
)
@@ -1,10 +1,17 @@
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
# SPDX-License-Identifier: Apache-2.0
import os
from dataclasses import dataclass, field
from typing import Any
import numpy as np
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
def _wan_1_3b_coefficients(p: TeaCacheParams) -> list[float]:
@@ -267,6 +274,160 @@ class Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams):
)
@dataclass
class Wan_Animate_2_14B_SamplingParam(Wan2_2_Base_SamplingParams):
"""Sampling defaults for Wan-Animate-2 14B."""
# Single expert: no guidance_scale_2 / boundary switching.
guidance_scale: float = 3.0
num_inference_steps: int = 40 # official README demo default
# The reference video is resampled to this rate and the output MP4 is written at it.
fps: int = 16
# Carry the reference video's audio track on the output, as the official pipeline does:
# the track is decoded in-process and the shared MP4 writer encodes it. False gives a
# silent video. A track that cannot be decoded is logged and the output stays silent.
enable_audio: bool = True
# Per-clip denoising length, 4k+1 (VAE stride 4); larger = better temporal coherence
# but more VRAM. Output length follows the reference video, not this.
clip_len: int = 37
# Text for the reference-video branch (prompt_ref_embeds). The official demo conditions it on a
# fixed description of the reference video, not on the character caption.
prompt_ref: str = "人物动作的参考视频"
# The reference image comes via the base image_path and the reference video via the
# base video_path; neither needs a model-specific field.
# (width, height) pairs; upstream Wan-Animate default is 640x800.
supported_resolutions: list[tuple[int, int]] | None = field(
default_factory=lambda: [
(640, 800), # upstream Wan-Animate default
(720, 1280), # 9:16
]
)
def __post_init__(self) -> None:
# The in-context denoising loop runs every block at every step; the shared DiT's
# cache heuristics are never consulted.
if self.enable_teacache or self.enable_spectrum:
raise ValueError(
"Wan-Animate-2 does not support enable_teacache or enable_spectrum."
)
requested_clip_len = self.clip_len
self.clip_len = _round_clip_len(self.clip_len)
if self.clip_len < MIN_CLIP_LEN:
raise ValueError(
f"clip_len={requested_clip_len!r} is not supported: a clip is 4k+1 frames "
f"with k >= 1, so clip_len must be an int >= {MIN_CLIP_LEN}"
)
if self.clip_len != requested_clip_len:
logger.info(
"clip_len=%s is not 4k+1; using clip_len=%s",
requested_clip_len,
self.clip_len,
)
# The output length follows the reference video and the stages read clip_len, so
# `num_frames` is not a length control here; the multi-GPU frame alignment must
# leave it alone, and it must stay writable so the warmup probe can shrink.
self.adjust_frames = False
super().__post_init__()
@classmethod
def video_request_extra_fields(cls) -> frozenset[str]:
# Settable over the /v1/videos API: the per-clip length, the reference-branch
# text (the official gradio exposes prompt_ref as well) and the audio track. The
# reference video is the base video_path field, so it needs no entry here.
return frozenset({"clip_len", "prompt_ref", "enable_audio"})
def prepare_synthetic_warmup_request_for_queue(
self, req: Any, server_args: Any
) -> None:
"""Give the synthetic warmup request the driving video the generic builder does not
supply, sized so the request denoises exactly one clip."""
del server_args
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
single_clip_reference_video_len,
write_reference_video,
)
image_path = req.image_path
if isinstance(image_path, list):
image_path = image_path[0] if image_path else None
if not isinstance(image_path, str) or not image_path:
raise ValueError(
"Wan-Animate-2 synthetic warmup requires the warmup reference image"
)
# The builder sizes warmup by num_frames (the class default or its bounded cap), but
# the stage reads clip_len and the compiled attention is specialized per clip
# geometry, so warm up at the production clip length unless more frames were asked.
clip_len = _round_clip_len(max(int(req.num_frames), self.clip_len))
num_video_frames = single_clip_reference_video_len(clip_len)
fps = int(req.fps)
# Same directory and lifetime as the warmup image; the name carries the frame count
# because every warmup request is built before the first one runs.
video_path = os.path.join(
os.path.dirname(image_path),
f"warmup_driving_video_{num_video_frames}f_{fps}fps.mp4",
)
write_reference_video(
video_path,
_synthetic_driving_video_frames(num_video_frames),
fps=fps,
)
req.video_path = video_path
req.clip_len = clip_len
req.num_frames = clip_len
# The synthetic driving video has no audio track.
req.enable_audio = False
def _adjust_visual_fields(self, server_args: Any, pipeline_config: Any) -> None:
super()._adjust_visual_fields(server_args, pipeline_config)
if self.num_frames != type(self).num_frames:
logger.info(
"Wan-Animate-2 ignores num_frames=%s (the /v1/videos `seconds`): the output "
"length follows the reference video; clip_len=%s is the per-clip chunk",
self.num_frames,
self.clip_len,
)
# Smallest clip: one conditioning frame plus one VAE temporal stride (4k+1, k >= 1).
MIN_CLIP_LEN = 5
def _round_clip_len(clip_len: int) -> int:
"""Clips are 4k+1 pixel frames (Wan VAE temporal stride 4). Other values map to the
4k+1 with the same k: 4k rounds up by one frame, 4k+2 and 4k+3 round down."""
if clip_len % 4 != 1:
clip_len = clip_len // 4 * 4 + 1
return clip_len
# Pixel size of the synthetic warmup driving video; the stage rescales every frame to the
# request area anyway, so this only has to be a size the shared video writer takes as is
# and cheap to encode.
_WARMUP_DRIVING_VIDEO_SIZE = 64
def _synthetic_driving_video_frames(num_frames: int) -> np.ndarray:
"""``[num_frames, 64, 64, 3]`` uint8 frames: a gradient that drifts one row per frame,
so consecutive frames differ like real motion."""
size = _WARMUP_DRIVING_VIDEO_SIZE
ramp = np.linspace(0, 255, size, dtype=np.float32)
rows = np.broadcast_to(ramp[:, None], (size, size))
cols = np.broadcast_to(ramp[None, :], (size, size))
frames = np.empty((num_frames, size, size, 3), dtype=np.uint8)
for index in range(num_frames):
frames[index, ..., 0] = np.roll(rows, index, axis=0)
frames[index, ..., 1] = cols
frames[index, ..., 2] = 127
return frames
@dataclass
class Turbo_Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams):
guidance_scale: float = 3.5 # high_noise
@@ -710,6 +710,9 @@ class WeightOverrideComponentLoader(ComponentLoader):
"""Base for loaders that consume an exact weights-only override."""
ignored_checkpoint_prefixes: tuple[str, ...] = ()
# True: a checkpoint key that maps to no model parameter fails the load instead
# of being logged and skipped.
strict_checkpoint_keys: bool = False
def load_state_dict_model(
self,
@@ -738,7 +741,7 @@ class WeightOverrideComponentLoader(ComponentLoader):
fsdp_inference=server_args.should_use_fsdp_for_component(component_name),
param_dtype=dtype,
reduce_dtype=torch.float32,
strict=False,
strict=self.strict_checkpoint_keys,
weight_load_plan=weight_load_plan,
checkpoint_key_filter=checkpoint_key_filter,
weights_iterator=weights_iterator,
@@ -214,22 +214,28 @@ def fit_auto_residency_probe(
floor_units = min((record.workload_units() for record in records), default=0)
fitted, steps = req, 0
while True:
units = (
max(1, int(fitted.width or 1))
* max(1, int(fitted.height or 1))
* max(1, int(fitted.num_frames or 1))
)
units = _probe_workload_units(fitted)
estimate = estimate_default_workload_peak_bytes(
records=records, target_units=units
)
if estimate is None or estimate <= budget or units <= floor_units:
return fitted, estimate, steps
lighter = lighten_warmup_req(server_args, fitted)
if lighter is None:
# Sampling params that pin the frame count (Wan-Animate-2 mirrors clip_len
# in __post_init__) hand back an equal-size req; treat that as the floor.
if lighter is None or _probe_workload_units(lighter) >= units:
return fitted, estimate, steps
fitted, steps = lighter, steps + 1
def _probe_workload_units(req: Req) -> int:
return (
max(1, int(req.width or 1))
* max(1, int(req.height or 1))
* max(1, int(req.num_frames or 1))
)
class GPUWorker(GPUWorkerPostTrainingMixin):
"""
A worker that executes the model on a single GPU.
@@ -0,0 +1,445 @@
# SPDX-License-Identifier: Apache-2.0
#
# Wan-Animate-2 DiT. Subclasses WanTransformer3DModel so TP / Ulysses SP shard it
# unchanged; only the per-block self-attention is swapped for the in-context mechanism
# (build_reference_kv runs forward_ref once per clip and returns each block's rotated reference-video
# K/V, forward_gen attends over [gen tokens, then that reference-video K/V]). No extra parameters.
# Names kept from the reference implementation: the forward_ref / forward_gen methods and the
# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image.
from collections import OrderedDict
from typing import Any
import torch
import torch.nn as nn
from torch.nn.attention.flex_attention import BlockMask, create_block_mask
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
WanAnimate2Config,
)
from sglang.multimodal_gen.runtime.distributed import get_sp_world_size
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
gather_seq,
shard_seq,
)
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
WanAnimate2TransformerBlock,
_InContextLayout,
_reference_video_key_mask,
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
WanAnimate2ClipConditioning,
WanAnimate2ReferenceKV,
)
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformer3DModel
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
class WanAnimate2Transformer3DModel(WanTransformer3DModel):
"""Wan-Animate-2 DiT (in-context variant) on the Wan2.2-I2V-14B backbone.
Attention goes through USPAttention, so callers must wrap the forward in
``set_forward_context(...)``.
"""
# Registry alias: the diffusers class name of this transformer.
_aliases = ["WanAnimate2Transformer3DModel"]
param_names_mapping = WanAnimate2Config().param_names_mapping
reverse_param_names_mapping = WanAnimate2Config().reverse_param_names_mapping
lora_param_names_mapping = WanAnimate2Config().lora_param_names_mapping
def __init__(
self,
config: WanAnimate2Config,
hf_config: dict[str, Any] | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__(
config=config, hf_config=hf_config or {}, quant_config=quant_config
)
# RoPE placement of the reference-video tokens relative to the gen grid; plain
# ints from the config, not parameters, so the Wan state dict is unchanged.
self.refer_offset_t = config.refer_offset_t
self.refer_offset_h = config.refer_offset_h
# <0: place reference-video tokens right of the gen grid (w offset = gen W in tokens).
self.refer_offset_w = config.refer_offset_w
self.refer_stride = config.refer_stride
# Prompt embeddings are padded/truncated to this many tokens.
self.max_text_len = config.text_len
self._block_masks: OrderedDict[tuple[int, int, int, str], BlockMask] = (
OrderedDict()
)
# RoPE tables keyed by the (f, h, w) position ranges and device; see _rope_tables.
self._rope_cache: OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]] = (
OrderedDict()
)
def _make_block(
self,
*,
index: int,
config: WanAnimate2Config,
quant_config: QuantizationConfig | None,
) -> nn.Module:
# Same constructor args as the Wan block and no new params, so the Wan checkpoint
# mapping and TP sharding apply unchanged.
return WanAnimate2TransformerBlock(
config.num_attention_heads * config.attention_head_dim,
config.ffn_dim,
config.num_attention_heads,
config.qk_norm,
config.cross_attn_norm,
config.eps,
config.added_kv_proj_dim,
self._supported_attention_backends
| {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
prefix=f"blocks.{index}",
attention_type=config.attention_type,
sla_topk=config.sla_topk,
quant_config=quant_config,
log_scale=config.log_scale,
)
# Device and dtype of the resident weights; the patch embedding is never offloaded.
@property
def _device(self) -> torch.device:
return self.patch_embedding.proj.weight.device
@property
def _dtype(self) -> torch.dtype:
return self.patch_embedding.proj.weight.dtype
def _rope_tables(
self,
f_range: tuple[int, int, int], # (start, length, stride)
h_range: tuple[int, int], # (start, length)
w_range: tuple[int, int],
) -> tuple[torch.Tensor, torch.Tensor]:
"""RoPE ``(cos, sin)``, each ``[f*h*w, D//2]`` fp32, for the full grid.
Row i holds the position of token i, flattened in (f, h, w) row-major order
like the patch embedding output.
The table is cached per ranges."""
key = (f_range, h_range, w_range, str(self._device))
if key in self._rope_cache:
self._rope_cache.move_to_end(key)
return self._rope_cache[key]
# Bound retained GPU tables; evict before allocating the next geometry.
if len(self._rope_cache) >= 4:
self._rope_cache.popitem(last=False)
f_start, f_len, f_stride = f_range
f_idx = torch.arange(f_len, device=self._device) * f_stride + f_start
h_idx = torch.arange(h_range[1], device=self._device) + h_range[0]
w_idx = torch.arange(w_range[1], device=self._device) + w_range[0]
# Positions are built explicitly (not via the rank-sharding forward_from_grid) because
# the in-context blocks all-to-all-gather the full sequence before applying RoPE.
ff, hh, ww = torch.meshgrid(f_idx, h_idx, w_idx, indexing="ij")
positions = torch.stack([ff.reshape(-1), hh.reshape(-1), ww.reshape(-1)], dim=1)
cos, sin = self.rotary_emb.forward_uncached(positions.long())
tables = (cos.float(), sin.float())
self._rope_cache[key] = tables
return tables
def _generation_video_rope(
self, f: int, h: int, w: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""RoPE tables for the generated video's ``f x h x w`` token grid; positions start at 0."""
return self._rope_tables((0, f, 1), (0, h), (0, w))
def _reference_video_rope(
self, f: int, h: int, w: int, generation_video_w: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""RoPE tables for the reference video's ``f x h x w`` token grid, shifted by the
``refer_*`` offsets. When ``refer_offset_w`` is -1, the reference-video grid is placed
directly to the right of the generated grid (width offset = ``generation_video_w``)."""
offset_w = (
generation_video_w if self.refer_offset_w == -1 else self.refer_offset_w
)
return self._rope_tables(
(self.refer_offset_t, f, self.refer_stride),
(self.refer_offset_h, h),
(offset_w, w),
)
def _get_block_mask_from_layout(self, layout: _InContextLayout) -> BlockMask:
"""Cached flex_attention ``BlockMask`` over the padded [gen | reference-video] layout."""
key = (
layout.max_generation_video_num_tokens,
layout.max_reference_video_num_tokens,
layout.max_tokens_per_frame,
str(self._device),
)
if key in self._block_masks:
self._block_masks.move_to_end(key)
return self._block_masks[key]
if len(self._block_masks) >= 2:
self._block_masks.popitem(last=False)
# _compile=True keeps the block-sparse layout consistent with the compiled
# flex_attention; built once per key, so the compile cost is paid once.
block_mask = create_block_mask(
layout.block_mask_fn(),
B=None,
H=None,
Q_LEN=layout.padded_generation_video_len,
KV_LEN=layout.padded_kv_len,
device=self._device,
_compile=True,
)
self._block_masks[key] = block_mask
return block_mask
def _patch_embeddings(
self,
latent_36ch: torch.Tensor, # [B, 36, f, H, W]
) -> tuple[torch.Tensor, tuple[int, int, int]]:
"""Patch-embed the latent into the token stream ``[B, f*h*w, C]`` and return the
token grid_size ``(f, h, w)``; tokens are flattened row-major, w fastest."""
p_t, p_h, p_w = self.patch_size
_, _, num_frames, height, width = latent_36ch.shape
grid_size = (num_frames // p_t, height // p_h, width // p_w)
hidden_states = self.patch_embedding(latent_36ch)
hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous()
return hidden_states, grid_size
def _time_embeddings(
self, timestep: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
timestep_embeddings = self.condition_embedder.time_embedder(timestep, None)
timestep_modulation = self.condition_embedder.time_modulation(
timestep_embeddings
).unflatten(1, (6, -1))
return timestep_embeddings, timestep_modulation
def prepare_context(
self,
prompt_embeddings: torch.Tensor,
image_embeddings: torch.Tensor | None,
) -> torch.Tensor:
"""Project one branch's timestep-invariant context; owned by the current clip."""
prompt_embeddings = self._pad_or_truncate_prompt_embeddings(prompt_embeddings)
context = self.condition_embedder.text_embedder(prompt_embeddings)
if image_embeddings is not None:
image_context = self.condition_embedder.image_embedder(image_embeddings)
context = torch.cat([image_context, context], dim=1)
return context.to(self._dtype)
def _pad_or_truncate_prompt_embeddings(
self, prompt_embeddings: torch.Tensor
) -> torch.Tensor:
"""Zero-pad or truncate the prompt embeddings (``[L, C]`` or ``[B, L, C]``) to
``[B, max_text_len, C]``, as the official model does; cross-attention attends the pad rows too."""
if prompt_embeddings.dim() == 2:
prompt_embeddings = prompt_embeddings.unsqueeze(0) # [L, C] -> [1, L, C]
batch_size, num_tokens, channels = prompt_embeddings.shape
if num_tokens < self.max_text_len:
pad = prompt_embeddings.new_zeros(
batch_size, self.max_text_len - num_tokens, channels
)
prompt_embeddings = torch.cat([prompt_embeddings, pad], dim=1)
elif num_tokens > self.max_text_len:
prompt_embeddings = prompt_embeddings[:, : self.max_text_len]
return prompt_embeddings.to(self._dtype)
@torch.no_grad()
def build_reference_kv(
self, *, clip_cond: WanAnimate2ClipConditioning
) -> WanAnimate2ReferenceKV:
"""One forward_ref pass over the clip's reference video, returning every block's rotated K and V.
Text- and timestep-independent, so one result serves every step and CFG branch of the
clip; the caller owns it and the DiT keeps no copy. Attention goes through USPAttention,
so callers must wrap this in ``set_forward_context(...)`` like ``forward``.
"""
# 36-channel reference-video input: fp32 latents (16) over the bf16 condition (20),
# both cast to the param dtype so the cat runs in bf16 (see _make_generation_input).
reference_video_latent_36ch = torch.cat(
[
clip_cond.reference_video_latents.to(self._dtype),
clip_cond.reference_video_condition.to(self._dtype),
],
dim=0,
).unsqueeze(0) # [1, 36, cf, H', W']
reference_video_frame_0_image_embeddings = (
clip_cond.reference_video_frame_0_image_embeddings
) # [1, 257, 1280] bf16
# reference-video timestep is forced to 1.
reference_video_timestep = torch.ones(1, device=self._device, dtype=torch.long)
hidden_states, (reference_video_f, reference_video_h, reference_video_w) = (
self._patch_embeddings(reference_video_latent_36ch)
)
_, timestep_modulation = self._time_embeddings(reference_video_timestep)
reference_video_encoder_hidden_states = self.prepare_context(
clip_cond.prompt_ref_embeddings, reference_video_frame_0_image_embeddings
)
reference_video_rope_cos, reference_video_rope_sin = self._reference_video_rope(
reference_video_f,
reference_video_h,
reference_video_w,
generation_video_w=clip_cond.generation_video_grid_sizes[2],
)
reference_video_num_tokens = (
reference_video_f * reference_video_h * reference_video_w
)
# Ulysses SP: shard the reference-video sequence; each block a2a-gathers it and caches
# full-seq / head-shard K/V. The reference-video hidden states are discarded, so no gather.
# The gathered sequence is tail-padded to a multiple of the SP size; one key mask over
# that padding serves every block (None when nothing is padded).
reference_video_key_mask = None
sp_size = get_sp_world_size()
if sp_size > 1:
hidden_states, _ = shard_seq(hidden_states, dim=1)
gathered_seq_len = hidden_states.shape[1] * sp_size
if gathered_seq_len != reference_video_num_tokens:
reference_video_key_mask = _reference_video_key_mask(
batch_size=hidden_states.shape[0],
seq_len=gathered_seq_len,
num_valid=reference_video_num_tokens,
device=hidden_states.device,
)
k_by_block: dict[int, torch.Tensor] = {}
v_by_block: dict[int, torch.Tensor] = {}
# Call via __call__ (not forward_ref) so the layerwise-offload hooks fire.
for idx, block in enumerate(self.blocks):
hidden_states = block(
kv_cache_mode="ref",
hidden_states=hidden_states,
encoder_hidden_states=reference_video_encoder_hidden_states,
timestep_modulation=timestep_modulation,
reference_video_rope_cos=reference_video_rope_cos,
reference_video_rope_sin=reference_video_rope_sin,
reference_video_num_tokens=reference_video_num_tokens,
index=idx,
k_cache=k_by_block,
v_cache=v_by_block,
reference_video_key_mask=reference_video_key_mask,
)
return WanAnimate2ReferenceKV(k=k_by_block, v=v_by_block)
@torch.no_grad()
def forward(
self,
hidden_states: torch.Tensor, # [16, f, h, w] gen latent (B=1)
encoder_hidden_states: torch.Tensor, # this CFG branch's prompt embeddings [L, C]
timestep: torch.Tensor, # [1]
# [1, 257, 1280] bf16 CLIP embed of the reference image
encoder_hidden_states_image: torch.Tensor | None = None,
guidance: torch.Tensor | None = None,
*,
# This clip's K/V from build_reference_kv; the same object for every step and CFG branch.
reference_kv: WanAnimate2ReferenceKV,
# This clip's conditioning from build_clip_conditioning.
clip_cond: WanAnimate2ClipConditioning,
projected_context: torch.Tensor | None = None,
# The unconditional CFG branch skips block 9 (see _run_generation_blocks).
is_unconditional: bool = False,
) -> torch.Tensor:
sp_size = get_sp_world_size()
latent_36ch = self._make_generation_input(
hidden_states, clip_cond.generation_condition
)
hidden_states, (f, h, w) = self._patch_embeddings(latent_36ch)
timestep_embeddings, timestep_modulation = self._time_embeddings(timestep)
generation_encoder_hidden_states = (
projected_context
if projected_context is not None
else self.prepare_context(
encoder_hidden_states, encoder_hidden_states_image
)
)
layout = _InContextLayout(
(f, h, w),
clip_cond.reference_video_grid_sizes,
clip_cond.full_clip_grid_sizes,
)
# No seq padding here: hidden_states is exactly f*h*w tokens and the in-context
# layout pads internally. Under Ulysses SP each block a2a-gathers the full
# sequence, so shard tail padding lands at the global tail where forward_gen
# expects it.
if sp_size > 1:
hidden_states, _ = shard_seq(hidden_states, dim=1)
hidden_states = self._run_generation_blocks(
hidden_states,
encoder_hidden_states=generation_encoder_hidden_states,
timestep_modulation=timestep_modulation,
layout=layout,
reference_kv=reference_kv,
is_unconditional=is_unconditional,
)
if sp_size > 1:
# Gather the shards and trim tail padding back to the clip's token count.
hidden_states = gather_seq(
hidden_states, layout.generation_video_num_tokens, dim=1
)
return self._unpatchify(hidden_states, timestep_embeddings, (f, h, w))
def _make_generation_input(
self,
latent: torch.Tensor, # [16, f, h, w] fp32
generation_condition: torch.Tensor, # [20, f, h, w] bf16
) -> torch.Tensor:
"""36-channel DiT input ``[1, 36, f, h, w]``: the fp32 noisy latent over the bf16 generation condition."""
# Both cast to the param dtype so the cat runs in bf16 and does not rely on autocast;
# the fp32 latent would be rounded to bf16 by the patch embedding anyway.
return torch.cat(
[latent.to(self._dtype), generation_condition.to(self._dtype)], dim=0
).unsqueeze(0)
def _run_generation_blocks(
self,
hidden_states: torch.Tensor,
*,
encoder_hidden_states: torch.Tensor,
timestep_modulation: torch.Tensor,
layout: _InContextLayout,
reference_kv: WanAnimate2ReferenceKV,
is_unconditional: bool,
) -> torch.Tensor:
"""forward_gen through every block with the in-context RoPE tables and block mask."""
generation_video_rope_cos, generation_video_rope_sin = (
self._generation_video_rope(
layout.generation_video_f,
layout.generation_video_h,
layout.generation_video_w,
)
)
block_mask = self._get_block_mask_from_layout(layout)
# Call via __call__ (not forward_gen) so the layerwise-offload hooks fire.
for idx, block in enumerate(self.blocks):
# Block 9 is dropped on the unconditional branch, as in the reference
# wan_animate_2_model.forward_gen; the authors give no rationale.
if is_unconditional and idx == 9:
continue
hidden_states = block(
kv_cache_mode="gen",
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
timestep_modulation=timestep_modulation,
gen_cos=generation_video_rope_cos,
gen_sin=generation_video_rope_sin,
block_mask=block_mask,
layout=layout,
index=idx,
k_cache=reference_kv.k,
v_cache=reference_kv.v,
)
return hidden_states
EntryClass = WanAnimate2Transformer3DModel
@@ -0,0 +1,532 @@
# SPDX-License-Identifier: Apache-2.0
#
# Wan-Animate-2 DiT block: in-context self-attention over [gen tokens, then cached
# reference-video K/V], its RoPE / Ulysses helpers, and the per-clip _InContextLayout
# geometry.
# Names kept from the reference implementation: the forward_ref / forward_gen methods and the
# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image.
from __future__ import annotations
import math
from collections.abc import Callable
from functools import lru_cache, partial
from typing import Any, Literal
import torch
from torch.nn.attention.flex_attention import BlockMask, flex_attention
from sglang.multimodal_gen.runtime.distributed import (
get_sp_world_size,
)
from sglang.multimodal_gen.runtime.layers.usp import (
_usp_input_all_to_all_qkv as _sp_gather_qkv,
)
from sglang.multimodal_gen.runtime.layers.usp import (
_usp_output_all_to_all,
)
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformerBlock
# Eager flex_attention materializes the full [Q, KV] scores over the padded [gen | reference-video]
# sequence (tens of GiB at video resolution); compiled, it is one fused kernel.
_flex_attention_compiled = torch.compile(
flex_attention,
dynamic=False,
mode="max-autotune",
fullgraph=True,
backend="inductor",
)
def _apply_rope_interleaved(
x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
) -> torch.Tensor:
"""Interleaved (GPT-J style) RoPE, identical to ``_apply_rotary_emb(is_neox_style=False)``;
``x`` is ``[B, S, H, D]``, ``cos`` / ``sin`` are ``[S, D//2]`` fp32."""
cos = cos[None, :, None, :] # [1, S, 1, D//2]
sin = sin[None, :, None, :]
x1 = x[..., 0::2].float()
x2 = x[..., 1::2].float()
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
out = torch.stack([o1, o2], dim=-1).flatten(-2)
return out.type_as(x)
def _score_mod_impl(
score: torch.Tensor,
b_idx: torch.Tensor,
h_idx: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
hw: int,
log_scale: float,
) -> torch.Tensor:
# Biases the keys of generation frame 1, [hw, 2*hw) in the padded layout, exactly as the
# official Wan-Animate-2 code does; non-zero only when the checkpoint config sets log_scale.
condition = (kv_idx >= hw) & (kv_idx < 2 * hw)
return torch.where(condition, score + log_scale, score)
@lru_cache(maxsize=None)
def _make_score_mod(hw: int, log_scale: float) -> Callable[..., torch.Tensor]:
# Stable object identity per (hw, log_scale): compiled flex_attention guards on the
# score_mod object, so a fresh closure per call would recompile every clip.
return partial(_score_mod_impl, hw=hw, log_scale=log_scale)
def _sp_scatter_out(attn_output: torch.Tensor) -> torch.Tensor:
"""Ulysses output a2a: [B, S_full, N_local, C] -> [B, S_local, N_tp, C]."""
return _usp_output_all_to_all(attn_output, head_dim=2)
def _reference_video_key_mask(
*, batch_size: int, seq_len: int, num_valid: int, device: torch.device
) -> torch.Tensor:
"""``[B, S]`` bool key mask: True for the leading ``num_valid`` reference-video tokens."""
key_mask = torch.zeros((batch_size, seq_len), dtype=torch.bool, device=device)
key_mask[:, :num_valid] = True
return key_mask
class WanAnimate2TransformerBlock(WanTransformerBlock):
"""Wan block with the Wan-Animate-2 in-context self-attention.
No new parameters: forward_ref / forward_gen reuse the inherited modules and only
route the self-attention differently.
"""
def __init__(self, *args: Any, log_scale: float = 0.0, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
if self.use_offline_qk_rotation:
raise ValueError(
"Wan-Animate-2 blocks do not apply the offline Q/K rotation that "
"SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION selects for this checkpoint."
)
# Additive score bias on the keys of generation frame 1 (_score_mod_impl); 0.0 skips it.
self.log_scale = log_scale
def _modulate(
self, timestep_modulation: torch.Tensor
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
torch.Tensor,
]:
"""14B modulation split of ``timestep_modulation`` ``[B, 6, C]`` into six ``[B, 1, C]`` fp32 chunks; mirrors wanvideo.py."""
assert timestep_modulation.dim() == 3, (
"WanAnimate2TransformerBlock expects the 14B timestep modulation [B, 6, inner]; the ti2v "
"seq-len modulation path is not used by Wan-Animate-2."
)
e = self.scale_shift_table + timestep_modulation.float()
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(
6, dim=1
)
return shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa
def forward(
self, *, kv_cache_mode: Literal["ref", "gen"], **kwargs: Any
) -> torch.Tensor:
"""Dispatch to forward_ref / forward_gen through ``nn.Module.__call__`` so the
layerwise-offload module hooks fire for both passes; keyword arguments only."""
if kv_cache_mode == "ref":
return self.forward_ref(**kwargs)
if kv_cache_mode == "gen":
return self.forward_gen(**kwargs)
raise ValueError(
f"WanAnimate2TransformerBlock: unknown kv_cache_mode {kv_cache_mode!r} "
"(expected 'ref' or 'gen')."
)
def forward_ref(
self,
*,
hidden_states: torch.Tensor, # [B, S_reference_video, C] reference-video latent stream
encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] reference-video context (CLIP tokens, then text)
timestep_modulation: torch.Tensor, # [B, 6, C], reference-video timestep forced to 1
reference_video_rope_cos: torch.Tensor, # reference-video RoPE cos [reference_video_num_tokens, D//2]
reference_video_rope_sin: torch.Tensor,
reference_video_num_tokens: int, # reference_video_f * reference_video_h * reference_video_w (valid reference-video tokens)
index: int,
k_cache: dict[int, torch.Tensor],
v_cache: dict[int, torch.Tensor],
# [B, S_reference_video] key mask for the SP tail padding, built once per clip by the
# caller; None when the gathered sequence has no padding.
reference_video_key_mask: torch.Tensor | None = None,
) -> torch.Tensor:
"""Cache this block's rotated reference-video K and V, then run the reference-video self-attention
(reference-video RoPE, pad keys masked) and return the updated reference-video stream."""
orig_dtype = hidden_states.dtype
shift, scale, gate, c_shift, c_scale, c_gate = self._modulate(
timestep_modulation
)
norm_hidden_states = self.norm1(hidden_states, shift, scale)
query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
# Ulysses SP: gather the full reference-video sequence / shard heads before caching, so the
# cache matches forward_gen's post-a2a layout.
sp_size = get_sp_world_size()
if sp_size > 1:
query, key, value = _sp_gather_qkv(query, key, value)
# Reference-video self-attention. Under SP, S_reference_video is the gathered
# (tail-padded) length: RoPE the valid slice and mask the pad keys. Without padding the
# whole sequence is valid and no mask is passed: USPAttention serves a dense key mask
# through SDPA instead of the configured backend.
reference_video_seq_len = query.shape[1]
if reference_video_seq_len == reference_video_num_tokens:
q_roped = _apply_rope_interleaved(
query, reference_video_rope_cos, reference_video_rope_sin
)
k_roped = _apply_rope_interleaved(
key, reference_video_rope_cos, reference_video_rope_sin
)
key_mask = None
else:
q_roped = query.clone()
k_roped = key.clone()
q_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved(
query[:, :reference_video_num_tokens],
reference_video_rope_cos,
reference_video_rope_sin,
)
k_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved(
key[:, :reference_video_num_tokens],
reference_video_rope_cos,
reference_video_rope_sin,
)
key_mask = reference_video_key_mask
if key_mask is None:
key_mask = _reference_video_key_mask(
batch_size=hidden_states.shape[0],
seq_len=reference_video_seq_len,
num_valid=reference_video_num_tokens,
device=hidden_states.device,
)
# Reference positions are invariant within a clip. Keep the rotated key in the
# original cache storage; SP's value can share the gathered QKV allocation.
k_cache[index] = key.copy_(k_roped)
v_cache[index] = value
if sp_size > 1:
# q/k/v are already gathered: skip USPAttention's own a2a, then scatter back.
attn_output = self.attn1(
q_roped,
k_roped,
value,
attn_mask=key_mask,
skip_sequence_parallel_override=True,
)
attn_output = _sp_scatter_out(attn_output)
else:
attn_output = self.attn1(q_roped, k_roped, value, attn_mask=key_mask)
attn_output = attn_output.flatten(2)
return self._cross_and_ffn(
hidden_states,
attn_output,
encoder_hidden_states,
gate,
c_shift,
c_scale,
c_gate,
orig_dtype,
)
def forward_gen(
self,
*,
hidden_states: torch.Tensor, # [B, seq_len, C] gen stream (padded)
encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] gen context (CLIP tokens, then text)
timestep_modulation: torch.Tensor, # [B, 6, C]
gen_cos: torch.Tensor, # gen-RoPE cos [generation_video_num_tokens, D//2]
gen_sin: torch.Tensor,
block_mask: BlockMask, # per full-length clip grid
layout: _InContextLayout,
index: int,
k_cache: dict[int, torch.Tensor],
v_cache: dict[int, torch.Tensor],
) -> torch.Tensor:
"""One generation block with in-context masked self-attention over
[gen tokens, then cached reference-video]."""
orig_dtype = hidden_states.dtype
shift, scale, gate, c_shift, c_scale, c_gate = self._modulate(
timestep_modulation
)
norm_hidden_states = self.norm1(hidden_states, shift, scale)
query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
# Ulysses SP: gather the full gen sequence / shard heads. The cached reference-video K/V are
# already in this layout.
sp_size = get_sp_world_size()
if sp_size > 1:
query, key, value = _sp_gather_qkv(query, key, value)
# gen-RoPE on the valid gen tokens only (padding kept unrotated).
num_generation_tokens = layout.generation_video_num_tokens
q_roped = _apply_rope_interleaved(
query[:, :num_generation_tokens], gen_cos, gen_sin
)
k_roped = _apply_rope_interleaved(
key[:, :num_generation_tokens], gen_cos, gen_sin
)
if query.shape[1] != num_generation_tokens:
q_roped = torch.cat([q_roped, query[:, num_generation_tokens:]], dim=1)
k_roped = torch.cat([k_roped, key[:, num_generation_tokens:]], dim=1)
# Reference-video K is already rotated by forward_ref, including SP tail padding.
reference_video_k = k_cache[index]
reference_video_v = v_cache[index]
attn_output = self._in_context_attention(
q_roped,
k_roped,
value,
reference_video_k,
reference_video_v,
block_mask,
layout,
)
if sp_size > 1:
# scatter the sequence back / gather heads.
attn_output = _sp_scatter_out(attn_output)
attn_output = attn_output.flatten(2)
return self._cross_and_ffn(
hidden_states,
attn_output,
encoder_hidden_states,
gate,
c_shift,
c_scale,
c_gate,
orig_dtype,
)
def _in_context_attention(
self,
q: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd)
k: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd)
v: torch.Tensor, # [B, seq_len, H, D]
reference_video_k: torch.Tensor, # [B, reference_video_seq_len, H, D] (reference-video RoPE'd)
reference_video_v: torch.Tensor, # [B, reference_video_seq_len, H, D]
block_mask: BlockMask,
layout: _InContextLayout,
) -> torch.Tensor:
"""Build the padded [gen | reference-video] layout and run compiled flex_attention.
The block-mask / score_mod are over sequence positions only, so TP composes
trivially, and under Ulysses SP the callers hand in the full sequence with sharded
heads, so the layout and mask are identical to sp=1."""
batch_size, _, num_heads, head_dim = q.shape
device, dtype = q.device, q.dtype
padded_generation_video_len = layout.padded_generation_video_len
padded_kv_len = layout.padded_kv_len
q_padding = q[:, layout.generation_video_num_tokens :]
q_padded = torch.zeros(
batch_size,
padded_generation_video_len,
num_heads,
head_dim,
device=device,
dtype=dtype,
)
k_padded = torch.zeros(
batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype
)
v_padded = torch.zeros(
batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype
)
num_latent_frames = layout.generation_video_f
latent_hw = layout.generation_video_tokens_per_frame
q_valid = q[:, : layout.generation_video_num_tokens].view(
batch_size, num_latent_frames, latent_hw, num_heads, head_dim
)
k_valid = k[:, : layout.generation_video_num_tokens].view(
batch_size, num_latent_frames, latent_hw, num_heads, head_dim
)
v_valid = v[:, : layout.generation_video_num_tokens].view(
batch_size, num_latent_frames, latent_hw, num_heads, head_dim
)
max_tokens_per_frame = layout.max_tokens_per_frame
q_padded[:, : num_latent_frames * max_tokens_per_frame].view(
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
)[:, :, :latent_hw] = q_valid
k_padded[:, : num_latent_frames * max_tokens_per_frame].view(
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
)[:, :, :latent_hw] = k_valid
v_padded[:, : num_latent_frames * max_tokens_per_frame].view(
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
)[:, :, :latent_hw] = v_valid
reference_video_f, reference_video_tokens_per_frame = (
layout.reference_video_f,
layout.reference_video_tokens_per_frame,
)
reference_video_k_valid = reference_video_k[
:, : layout.reference_video_num_tokens
].view(
batch_size,
reference_video_f,
reference_video_tokens_per_frame,
num_heads,
head_dim,
)
reference_video_v_valid = reference_video_v[
:, : layout.reference_video_num_tokens
].view(
batch_size,
reference_video_f,
reference_video_tokens_per_frame,
num_heads,
head_dim,
)
k_padded[
:,
padded_generation_video_len : padded_generation_video_len
+ reference_video_f * max_tokens_per_frame,
].view(
batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim
)[:, :, :reference_video_tokens_per_frame] = reference_video_k_valid
v_padded[
:,
padded_generation_video_len : padded_generation_video_len
+ reference_video_f * max_tokens_per_frame,
].view(
batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim
)[:, :, :reference_video_tokens_per_frame] = reference_video_v_valid
# At 0.0 the bias is an exact identity, so the kernel is not asked to apply it;
# otherwise the cached partial keeps the identity compiled flex guards on (_make_score_mod).
log_scale = float(self.log_scale)
_score_mod = (
None
if log_scale == 0.0
else _make_score_mod(max_tokens_per_frame, log_scale)
)
# flex_attention takes [B, H, L, D]; the default 1/sqrt(D) scale matches USPAttention.
attn_output_padded = _flex_attention_compiled(
q_padded.transpose(1, 2),
k_padded.transpose(1, 2),
v_padded.transpose(1, 2),
block_mask=block_mask,
score_mod=_score_mod,
).transpose(1, 2) # -> [B, padded_generation_video_len, N, C]
attn_output_valid = attn_output_padded[
:, : num_latent_frames * max_tokens_per_frame
].view(
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
)[:, :, :latent_hw]
attn_output_valid = attn_output_valid.reshape(
batch_size, num_latent_frames * latent_hw, num_heads, head_dim
)
if q_padding.shape[1] == 0:
return attn_output_valid
return torch.cat([attn_output_valid, q_padding], dim=1) # [B, seq_len, N, C]
class _InContextLayout:
"""Token geometry of the padded [generation | reference-video] in-context layout.
Three sets of sizes: this clip's token counts; those of a full-length clip, which every
clip is padded up to; and the full-length sizes rounded up to 128 for flex_attention.
The attention pads to the last set and the block mask is built from the same numbers,
so one compiled mask serves every clip of a request."""
def __init__(
self,
generation_video_grid: tuple[int, int, int],
reference_video_grid: tuple[int, int, int],
full_clip_grid: tuple[int, int, int],
) -> None:
f, h, w = generation_video_grid
reference_video_f, reference_video_h, reference_video_w = reference_video_grid
full_clip_f, full_clip_h, full_clip_w = full_clip_grid
self.generation_video_f, self.generation_video_h, self.generation_video_w = (
f,
h,
w,
)
self.reference_video_f, self.reference_video_h, self.reference_video_w = (
reference_video_f,
reference_video_h,
reference_video_w,
)
# This clip; shorter than full length only for the last clip of a video.
self.generation_video_tokens_per_frame = h * w
self.reference_video_tokens_per_frame = reference_video_h * reference_video_w
self.generation_video_num_tokens = f * h * w
self.reference_video_num_tokens = (
reference_video_f * reference_video_h * reference_video_w
)
# A full-length clip. The generation video carries one extra reference-image frame,
# the reference video does not.
self.max_tokens_per_frame = full_clip_h * full_clip_w
self.max_generation_video_num_tokens = full_clip_f * self.max_tokens_per_frame
self.max_reference_video_num_tokens = (
full_clip_f - 1
) * self.max_tokens_per_frame
# Padded to flex_attention's 128-token blocks; keys are [generation | reference video].
self.padded_generation_video_len = (
math.ceil(self.max_generation_video_num_tokens / 128) * 128
)
self.padded_reference_video_len = (
math.ceil(self.max_reference_video_num_tokens / 128) * 128
)
self.padded_kv_len = (
self.padded_generation_video_len + self.padded_reference_video_len
)
def block_mask_fn(self) -> Callable[..., torch.Tensor]:
"""Returns the block mask function based on the [gen | reference-video] in-context layout.
A valid gen query attends every valid gen key plus the reference-video keys of its own frame.
keys: [generation tokens | generation pad | reference tokens | reference pad]
0 padded_generation_video_len padded_kv_len
queries: [generation tokens | generation pad]
0 padded_generation_video_len
"""
tokens_per_frame = self.max_tokens_per_frame
num_generation_tokens = self.max_generation_video_num_tokens
num_reference_tokens = self.max_reference_video_num_tokens
reference_start = self.padded_generation_video_len
def _mask_function(
b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor
) -> torch.Tensor:
# Pad positions are not tokens; pad queries attend nothing.
q_is_generation_token = q_idx < num_generation_tokens
# Generation frame of the query; frame 0 is the reference image.
q_frame = q_idx // tokens_per_frame
# Every generation token attends every generation token.
kv_is_generation_token = kv_idx < num_generation_tokens
# Reference-video tokens are attended only by the queries of their own frame.
kv_reference_idx = kv_idx - reference_start
kv_is_reference_token = (kv_idx >= reference_start) & (
kv_reference_idx < num_reference_tokens
)
# +1: the reference video has no counterpart for the generation stream's
# reference-image frame 0.
kv_reference_frame = kv_reference_idx // tokens_per_frame + 1
kv_is_same_frame_reference_token = kv_is_reference_token & (
q_frame == kv_reference_frame
)
return q_is_generation_token & (
kv_is_generation_token | kv_is_same_frame_reference_token
)
return _mask_function
@@ -0,0 +1,67 @@
# SPDX-License-Identifier: Apache-2.0
"""Per-clip contracts between the Wan-Animate-2 stages and the DiT.
Lives on the model side so the DiT can type its ``clip_cond`` / ``reference_kv`` kwargs
without importing a pipeline stage: the before-denoising stage builds the conditioning, the
DiT's ``build_reference_kv`` turns it into the reference K/V the denoising stage hands to
every ``forward`` of that clip.
"""
import msgspec
import torch
class WanAnimate2ClipConditioning(msgspec.Struct, frozen=True, kw_only=True):
"""Per-clip conditioning consumed by the DiT's forward_ref / forward_gen."""
# Inputs for ``forward_ref`` (reference video = the motion source)
# VAE latents of this clip's reference-video frames,
# [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32.
reference_video_latents: torch.Tensor
# Generation token grid (latent_t, latent_h // 2, latent_w // 2): the DiT patchifies the
# latent with patch_size (1, 2, 2), so each spatial side halves.
generation_video_grid_sizes: tuple[int, int, int]
# CLIP tokens of the clip's first reference-video frame, [1, 257, 1280] bf16.
reference_video_frame_0_image_embeddings: torch.Tensor
# [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16.
# Channels: [0:4] given-frame mask, all ones (every reference frame is known);
# [4:20] VAE latent, the same tensor as reference_video_latents.
# Time: the clip's reference-video latent frames; no reference-image frame in front.
# forward_ref cats it under reference_video_latents (16 ch) for the 36-channel DiT input.
reference_video_condition: torch.Tensor
# T5 embedding of prompt_ref, [L_prompt_ref, 4096] bf16.
prompt_ref_embeddings: torch.Tensor
# Inputs for: ``forward_gen``
# [20, latent_t, latent_h, latent_w] bf16.
# Channels: [0:4] given-frame mask (1 = known pixels, 0 = to generate);
# [4:20] VAE latent of the given pixels, encoded zeros elsewhere.
# Time: frame 0 is the reference image (mask = all ones); frames 1..latent_t are the clip's
# own clip_latent_t frames, given only for the num_frames_conditioning frames fed
# back from the previous clip (none for clip 0).
# forward_gen cats it under the noise latent (16 ch) for the 36-channel DiT input.
generation_condition: torch.Tensor
# Reference-video token grid (reference_video_latent_t, _h // 2, _w // 2); same halving.
reference_video_grid_sizes: tuple[int, int, int]
# Token grid of a full-length clip (the configured clip_len); the attention layout and
# block mask are sized from it even on the shorter last clip, so one compiled mask
# serves the whole request.
full_clip_grid_sizes: tuple[int, int, int]
# Pixel-frame count of this clip (the last clip may be shorter than clip_len).
num_frames: int
# Seeded initial noise for the denoising loop, [16, latent_t, latent_h, latent_w] fp32.
init_noise: torch.Tensor
class WanAnimate2ReferenceKV(msgspec.Struct, frozen=True):
"""One clip's post-RoPE reference-video K and V, keyed by DiT block index.
Returned by ``WanAnimate2Transformer3DModel.build_reference_kv`` and owned by the caller
for that clip only; the DiT keeps no copy between calls.
"""
# [1, S_reference_video, local_heads, head_dim] in the DiT param dtype. Under Ulysses SP
# this is the full sequence with sharded heads, the layout forward_gen uses after its a2a.
k: dict[int, torch.Tensor]
v: dict[int, torch.Tensor]
@@ -617,6 +617,61 @@ class WanTransformerBlock(nn.Module):
self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
def _qkv(
self, norm_hidden_states: torch.Tensor, local_heads: int
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
query, _ = self.to_q(norm_hidden_states)
key, _ = self.to_k(norm_hidden_states)
value, _ = self.to_v(norm_hidden_states)
if self.norm_q is not None:
query = (
tensor_parallel_rms_norm(query, self.norm_q)
if self.tp_rmsnorm
else self.norm_q(query)
)
if self.norm_k is not None:
key = (
tensor_parallel_rms_norm(key, self.norm_k)
if self.tp_rmsnorm
else self.norm_k(key)
)
return tuple(
tensor.squeeze(1).unflatten(2, (local_heads, self.dim_head))
for tensor in (query, key, value)
)
def _cross_and_ffn(
self,
hidden_states: torch.Tensor,
attn_output: torch.Tensor,
encoder_hidden_states: torch.Tensor,
gate_msa: torch.Tensor,
c_shift_msa: torch.Tensor,
c_scale_msa: torch.Tensor,
c_gate_msa: torch.Tensor,
orig_dtype: torch.dtype,
) -> torch.Tensor:
attn_output, _ = self.to_out(attn_output)
attn_output = attn_output.squeeze(1)
null_shift = null_scale = torch.zeros(
(1,), device=hidden_states.device, dtype=hidden_states.dtype
)
norm_hidden_states, hidden_states = self.self_attn_residual_norm(
hidden_states, attn_output, gate_msa, null_shift, null_scale
)
norm_hidden_states = norm_hidden_states.to(orig_dtype)
hidden_states = hidden_states.to(orig_dtype)
attn_output = self.attn2(
norm_hidden_states, context=encoder_hidden_states, context_lens=None
)
norm_hidden_states, hidden_states = self.cross_attn_residual_norm(
hidden_states, attn_output, 1, c_shift_msa, c_scale_msa
)
norm_hidden_states = norm_hidden_states.to(orig_dtype)
hidden_states = hidden_states.to(orig_dtype)
ff_output = self.ffn(norm_hidden_states)
return self.mlp_residual(ff_output, c_gate_msa, hidden_states).to(orig_dtype)
def forward(
self,
hidden_states: torch.Tensor,
@@ -655,23 +710,7 @@ class WanTransformerBlock(nn.Module):
# 1. Self-attention
norm_hidden_states = self.norm1(hidden_states, shift_msa, scale_msa)
query, _ = self.to_q(norm_hidden_states)
key, _ = self.to_k(norm_hidden_states)
value, _ = self.to_v(norm_hidden_states)
if self.norm_q is not None:
if self.tp_rmsnorm:
query = tensor_parallel_rms_norm(query, self.norm_q)
else:
query = self.norm_q(query)
if self.norm_k is not None:
if self.tp_rmsnorm:
key = tensor_parallel_rms_norm(key, self.norm_k)
else:
key = self.norm_k(key)
query = query.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
key = key.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
value = value.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
# Apply rotary embeddings
cos, sin = freqs_cis
@@ -729,38 +768,16 @@ class WanTransformerBlock(nn.Module):
attn_output = self.attn1(query, key, value)
attn_output = attn_output.flatten(2)
attn_output, _ = self.to_out(attn_output)
attn_output = attn_output.squeeze(1)
null_shift = null_scale = torch.zeros(
(1,), device=hidden_states.device, dtype=hidden_states.dtype
return self._cross_and_ffn(
hidden_states,
attn_output,
encoder_hidden_states,
gate_msa,
c_shift_msa,
c_scale_msa,
c_gate_msa,
orig_dtype,
)
norm_hidden_states, hidden_states = self.self_attn_residual_norm(
hidden_states, attn_output, gate_msa, null_shift, null_scale
)
norm_hidden_states, hidden_states = (
norm_hidden_states.to(orig_dtype),
hidden_states.to(orig_dtype),
)
# 2. Cross-attention
attn_output = self.attn2(
norm_hidden_states, context=encoder_hidden_states, context_lens=None
)
norm_hidden_states, hidden_states = self.cross_attn_residual_norm(
hidden_states, attn_output, 1, c_shift_msa, c_scale_msa
)
norm_hidden_states, hidden_states = (
norm_hidden_states.to(orig_dtype),
hidden_states.to(orig_dtype),
)
# 3. Feed-forward
ff_output = self.ffn(norm_hidden_states)
hidden_states = self.mlp_residual(ff_output, c_gate_msa, hidden_states)
hidden_states = hidden_states.to(orig_dtype)
return hidden_states
class WanTransformerBlock_VSA(nn.Module):
@@ -1057,29 +1074,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
)
# 3. Transformer blocks
attn_backend = get_global_server_args().attention_backend
transformer_block = (
WanTransformerBlock_VSA
if (attn_backend and attn_backend.lower() == "video_sparse_attn")
else WanTransformerBlock
)
self.blocks = nn.ModuleList(
[
transformer_block(
inner_dim,
config.ffn_dim,
config.num_attention_heads,
config.qk_norm,
config.cross_attn_norm,
config.eps,
config.added_kv_proj_dim,
self._supported_attention_backends
| {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
prefix=f"blocks.{i}",
attention_type=config.attention_type,
sla_topk=config.sla_topk,
quant_config=quant_config,
)
self._make_block(index=i, config=config, quant_config=quant_config)
for i in range(config.num_layers)
]
)
@@ -1127,6 +1124,36 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.layer_names = ["blocks"]
def _make_block(
self,
*,
index: int,
config: WanVideoConfig,
quant_config: QuantizationConfig | None,
) -> nn.Module:
"""Block factory; subclasses override it to substitute their block class."""
attn_backend = get_global_server_args().attention_backend
transformer_block = (
WanTransformerBlock_VSA
if (attn_backend and attn_backend.lower() == "video_sparse_attn")
else WanTransformerBlock
)
return transformer_block(
config.num_attention_heads * config.attention_head_dim,
config.ffn_dim,
config.num_attention_heads,
config.qk_norm,
config.cross_attn_norm,
config.eps,
config.added_kv_proj_dim,
self._supported_attention_backends
| {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
prefix=f"blocks.{index}",
attention_type=config.attention_type,
sla_topk=config.sla_topk,
quant_config=quant_config,
)
@lru_cache(maxsize=1)
def _compute_rope_for_sequence_shard(
self,
@@ -1338,30 +1365,37 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
if seq_shard_pad > 0:
hidden_states = hidden_states[:, :seq_len_orig, :]
# 5. Output norm, projection & unpatchify
if temb.dim() == 3:
return self._unpatchify(
hidden_states,
temb,
(post_patch_num_frames, post_patch_height, post_patch_width),
)
def _unpatchify(
self,
hidden_states: torch.Tensor,
timestep_embeddings: torch.Tensor,
grid_size: tuple[int, int, int],
) -> torch.Tensor:
"""Output norm, projection and token-grid reconstruction for Wan variants."""
if timestep_embeddings.dim() == 3:
# batch_size, seq_len, inner_dim (wan 2.2 ti2v)
shift, scale = (
self.scale_shift_table.unsqueeze(0) + temb.unsqueeze(2)
self.scale_shift_table.unsqueeze(0) + timestep_embeddings.unsqueeze(2)
).chunk(2, dim=2)
shift = shift.squeeze(2)
scale = scale.squeeze(2)
else:
# batch_size, inner_dim
shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk(2, dim=1)
shift, scale = (
self.scale_shift_table + timestep_embeddings.unsqueeze(1)
).chunk(2, dim=1)
hidden_states = self.norm_out(hidden_states, shift, scale)
hidden_states, _ = self.proj_out(hidden_states)
hidden_states = hidden_states.reshape(
batch_size,
post_patch_num_frames,
post_patch_height,
post_patch_width,
p_t,
p_h,
p_w,
-1,
hidden_states.shape[0], *grid_size, *self.patch_size, -1
)
hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6)
output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3)
@@ -9,6 +9,7 @@
# This wrapper delegates all numerical work to diffusers' implementation
# and only adapts the interface for sglang's denoising stage.
import numpy as np
import torch
from diffusers import (
DPMSolverMultistepScheduler as DiffusersDPMSolverMultistepScheduler,
@@ -101,10 +102,43 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
def begin_index(self) -> int | None:
return self._begin_index
def set_timesteps(self, num_inference_steps: int, device=None, **kwargs):
self._inner.set_timesteps(num_inference_steps, device=device, **kwargs)
def set_timesteps(
self,
num_inference_steps: int | None = None,
device=None,
sigmas: np.ndarray | list[float] | None = None,
**kwargs,
):
if sigmas is None:
self._inner.set_timesteps(num_inference_steps, device=device, **kwargs)
else:
self._set_sigmas(np.asarray(sigmas, dtype=np.float64), device)
self.timesteps = self._inner.timesteps
def _set_sigmas(self, sigmas: np.ndarray, device=None) -> None:
inner = self._inner
if inner.config.final_sigmas_type == "sigma_min":
sigma_last = (
(1 - inner.alphas_cumprod[0]) / inner.alphas_cumprod[0]
) ** 0.5
else:
sigma_last = 0
timesteps = sigmas * inner.config.num_train_timesteps
sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32)
inner.sigmas = torch.from_numpy(sigmas)
inner.timesteps = torch.from_numpy(timesteps).to(
device=device, dtype=torch.int64
)
inner.num_inference_steps = len(timesteps)
self._reset_solver_state()
def _reset_solver_state(self) -> None:
inner = self._inner
inner.model_outputs = [None] * inner.config.solver_order
inner.lower_order_nums = 0
inner._step_index = None
inner._begin_index = None
def scale_model_input(
self, sample: torch.Tensor, timestep: int | None = None
) -> torch.Tensor:
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
"""Wan-Animate-2 pipeline.
Stages: WanAnimate2BeforeDenoisingStage (per-clip conditioning) -> WanAnimate2DenoisingStage
(per-clip denoise + in-loop VAE decode) -> WanAnimate2OutputStage (emit frames).
Routed by model_index.json ``_class_name == "WanAnimate2Pipeline"``; config comes from
the registry detector (Wan_Animate_2_14B_Config). Every component loads through the
standard component loaders. Text uses TextEncodingStage; CLIP and VAE adapters
preserve the reference preprocessing and latent scaling.
"""
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
TransformerLoader,
)
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2 import (
WanAnimate2BeforeDenoisingStage,
WanAnimate2DenoisingStage,
WanAnimate2ImageEncoderAdapter,
WanAnimate2OutputStage,
WanAnimate2VaeAdapter,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class WanAnimate2TransformerLoader(TransformerLoader):
"""The checkpoint is the plain Wan2.2-I2V-14B backbone, so every key must map."""
component_names = []
strict_checkpoint_keys = True
class WanAnimate2Pipeline(ComposedPipelineBase):
"""Wan-Animate-2 14B in-context reference image + reference video -> video pipeline."""
pipeline_name = "WanAnimate2Pipeline"
is_video_pipeline = True
# All seven model_index.json components. The scheduler is built from the
# checkpoint's scheduler/scheduler_config.json (DPM-Solver++, flow prediction,
# flow sigmas) by the shared loader. Two of its settings are overridden at
# runtime: the stages call set_timesteps(sigmas=...) with the official Wan
# sigma grid at every clip (sigma_0 is exactly 1.0), shifted by
# pipeline_config.flow_shift, so the JSON's flow_shift and timestep_spacing
# are not consulted. The JSON's beta_schedule only shapes the beta-derived
# tables, which the flow-sigma path does not read.
_required_config_modules: list[str] = [
"text_encoder",
"tokenizer",
"vae",
"transformer",
"image_encoder",
"image_processor",
"scheduler",
]
component_loaders = {"transformer": WanAnimate2TransformerLoader}
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
# Keep native modules visible to residency and layerwise-offload management;
# adapters only translate the model-specific stage API.
device = get_local_torch_device()
image_encoder = WanAnimate2ImageEncoderAdapter.from_loaded(
self.get_module("image_encoder"),
image_processor=self.get_module("image_processor"),
device=device,
)
vae = WanAnimate2VaeAdapter(self.get_module("vae"))
self.add_stage(
stage_name="wan_animate_2_before_denoising",
stage=WanAnimate2BeforeDenoisingStage(
vae=vae,
image_encoder=image_encoder,
text_encoder=self.get_module("text_encoder"),
tokenizer=self.get_module("tokenizer"),
pipeline_config=server_args.pipeline_config,
scheduler=self.get_module("scheduler"),
),
)
# Decode each clip before constructing the following clip's conditions.
self.add_stage(
stage_name="wan_animate_2_denoising",
stage=WanAnimate2DenoisingStage(
transformer=self.get_module("transformer"),
scheduler=self.get_module("scheduler"),
pipeline=self,
vae=vae,
image_encoder=image_encoder,
),
)
# Replaces the standard DecodingStage: frames are already decoded in the loop.
self.add_stage_factory(
RoleType.DECODER,
lambda: WanAnimate2OutputStage(),
"decoding_stage",
)
EntryClass = WanAnimate2Pipeline
@@ -234,8 +234,31 @@ class DecodingStage(PipelineStage):
Returns:
Decoded video tensor with shape (batch, channels, frames, height, width),
normalized to [0, 1] range and moved to CPU as float32
normalized to [0, 1] on the decode device.
"""
normalized_on_frames = (
(lambda frames: on_frames(_to_unit_range(frames)))
if on_frames is not None
else None
)
image = self.decode_raw(
latents,
server_args,
vae_dtype=vae_dtype,
on_frames=normalized_on_frames,
)
return _to_unit_range(image)
@torch.no_grad()
def decode_raw(
self,
latents: torch.Tensor,
server_args: ServerArgs,
*,
vae_dtype: torch.dtype,
on_frames=None,
) -> torch.Tensor:
"""Decode without pixel normalization, including for inter-clip conditioning."""
latents = latents.to(get_local_torch_device())
# The caller resolves the decode-only override before component use so
# residency and execution agree on the target dtype.
@@ -270,9 +293,7 @@ class DecodingStage(PipelineStage):
latents = latents.to(vae_dtype)
decode_kwargs = {}
if on_frames is not None:
decode_kwargs["on_frames"] = lambda frames: on_frames(
_to_unit_range(frames)
)
decode_kwargs["on_frames"] = on_frames
with temporary_module_dtype(
self.vae, vae_dtype, enabled=should_cast_vae
) as vae:
@@ -304,8 +325,6 @@ class DecodingStage(PipelineStage):
raise
image = _ensure_tensor_decode_output(decode_output)
# De-normalize image to [0, 1] range
image = _to_unit_range(image)
return image
def load_model(self):
@@ -2084,11 +2084,40 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
neg_cond_kwargs=ctx.neg_cond_kwargs,
guidance=ctx.guidance,
)
timesteps_cpu = self._run_denoising_loop(ctx, batch, server_args)
# collect rollout outputs before finalization gathers or replaces latents
if batch.rollout:
self._postprocess_rollout_outputs(
batch=batch,
latents=ctx.latents,
num_inference_steps=len(timesteps_cpu),
final_timestep=timesteps_cpu.new_zeros(()),
server_args=server_args,
)
self._finalize_denoising_loop(ctx, batch, server_args)
return batch
def _run_denoising_loop(
self,
ctx: DenoisingContext,
batch: Req,
server_args: ServerArgs,
*,
collect_trajectory: bool = True,
) -> torch.Tensor:
"""Run a prepared loop, including profiling and DiT residency.
Clip-based models can reuse this without finalizing the entire request.
"""
denoising_start_time = time.time()
self._before_denoising_loop(ctx, batch, server_args)
# to avoid device-sync caused by timestep comparison
timesteps_cpu = ctx.timesteps.cpu()
num_timesteps = timesteps_cpu.shape[0]
progress_step_interval = ctx.extra.get(
"progress_step_interval", ctx.scheduler.order
)
# Re-resolve the explicit-range gate so the per-step markers
# below honor this request's is_warmup state. Layer hooks are
# registered by the residency manager at the use-site.
@@ -2132,7 +2161,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
# BEFORE _run_denoising_step so ctx.latents is still the
# pre-step value. Gated on batch.rollout to keep the
# non-rollout path strictly untouched.
if batch.rollout:
if collect_trajectory and batch.rollout:
batch._rollout_loop_step_index = step_index
self._maybe_append_dit_trajectory_step(
batch=batch,
@@ -2141,11 +2170,12 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
step_index=step_index,
)
self._run_denoising_step(ctx, step, batch, server_args)
self._record_trajectory(ctx, step, batch, server_args)
if collect_trajectory:
self._record_trajectory(ctx, step, batch, server_args)
if step_index == num_timesteps - 1 or (
(step_index + 1) > ctx.num_warmup_steps
and (step_index + 1) % ctx.scheduler.order == 0
and (step_index + 1) % progress_step_interval == 0
and progress_bar is not None
):
progress_bar.update()
@@ -2165,20 +2195,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
del step
self._finish_active_component_use()
# Rollout postprocessing must run BEFORE _finalize_denoising_loop so
# the final scheduler.step output (ctx.latents) is still SP-sharded and
# can be gathered uniformly alongside the per-step dit_trajectory via
# gather_stacked_latents_for_sp.
if batch.rollout:
self._postprocess_rollout_outputs(
batch=batch,
latents=ctx.latents,
num_inference_steps=num_timesteps,
final_timestep=timesteps_cpu.new_zeros(()),
server_args=server_args,
)
self._finalize_denoising_loop(ctx, batch, server_args)
return batch
return timesteps_cpu
def _get_extra_func_kwarg_names(self, func) -> tuple[bool, frozenset[str]]:
import functools
@@ -0,0 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
"""Wan-Animate-2-specific pipeline stages."""
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
WanAnimate2BeforeDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
WanAnimate2DenoisingStage,
WanAnimate2OutputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
WanAnimate2ImageEncoderAdapter,
WanAnimate2VaeAdapter,
)
__all__ = [
"WanAnimate2BeforeDenoisingStage",
"WanAnimate2DenoisingStage",
"WanAnimate2OutputStage",
"WanAnimate2ImageEncoderAdapter",
"WanAnimate2VaeAdapter",
]
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
"""Reference-video audio extraction for the Wan-Animate-2 before-denoising stage."""
from __future__ import annotations
from collections.abc import Callable
import torch
from sglang.srt.multimodal.audio_from_video import decode_audio_container
# Logger callables with the ``PipelineStage.log_*`` signature: ``(msg, *args)``.
LogFn = Callable[..., None]
def extract_reference_video_audio(
reference_video_path: str,
*,
log_info: LogFn,
log_warning: LogFn,
log_no_audio_track: LogFn | None = None,
) -> tuple[torch.Tensor | None, int | None]:
"""Return ``(audio, sample_rate)`` for ``OutputBatch.audio``: audio ``[1, C, L]`` fp32 in
[-1, 1] (dim 0 is the per-output axis the save path indexes) at the track's native sample
rate and channel count, or ``(None, None)`` when the video has no decodable audio track.
The track is decoded in-process by the shared ``decode_audio_container`` reader (PyAV).
Best-effort: every failure is logged, none is raised. A video without an audio track goes
to ``log_no_audio_track`` (``log_warning`` when None); a track that cannot be decoded goes
to ``log_warning`` with the reader's message."""
try:
sample_rate = _audio_track_sample_rate(reference_video_path)
if sample_rate is None:
(log_no_audio_track or log_warning)(
"Wan-Animate-2: reference video %s has no audio track; the output is silent.",
reference_video_path,
)
return None, None
# [L, C] fp32 in [-1, 1]; the reader needs a target rate, so the native one keeps the
# track unresampled.
samples = decode_audio_container(
reference_video_path, target_sr=sample_rate, mono=False
)
except Exception as e: # audio is best-effort: a failure never fails the request
log_warning(
"Wan-Animate-2: cannot decode the audio track of reference video %s (%s); "
"the output is silent.",
reference_video_path,
e,
)
return None, None
audio = torch.from_numpy(samples).T.contiguous().unsqueeze(0) # [1, C, L]
log_info(
"Wan-Animate-2: extracted reference-video audio %s @ %d Hz.",
tuple(audio.shape),
sample_rate,
)
return audio, sample_rate
def _audio_track_sample_rate(path: str) -> int | None:
"""Native sample rate of the video's first audio stream; None when it has none."""
import av
with av.open(path) as container:
if not container.streams.audio:
return None
return int(container.streams.audio[0].rate)
@@ -0,0 +1,706 @@
# SPDX-License-Identifier: Apache-2.0
"""Wan-Animate-2 before-denoising stage.
Builds the per-clip conditioning container (``WanAnimate2ClipConditioning``) the DiT
reads via its ``clip_cond`` kwarg, plus the clip-invariant text/CLIP/VAE encodings.
Preprocessing helpers live in ``preprocess.py`` next to this module.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
import cv2
import msgspec
import numpy as np
import torch
from einops import rearrange
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
get_sp_world_size,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
WanAnimate2ClipConditioning,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
DPMSolverMultistepScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio import (
extract_reference_video_audio,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
LetterboxInfo,
get_padding_len,
make_conditioning_mask,
read_reference_video_frames,
resize_by_area,
validate_and_get_single_string,
zigzag_padding,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
if TYPE_CHECKING:
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
Wan_Animate_2_14B_Config,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
WanAnimate2ImageEncoderAdapter,
WanAnimate2VaeAdapter,
)
# Frames of the previous clip's decoded output that open the next clip as given (not denoised);
# the official pipelines/wan_animate_2_pipeline.py hard-codes num_frames_conditioning = 1.
_NUM_FRAMES_CONDITIONING = 1
# Wan VAE temporal compression: 4 pixel frames per latent frame after the first.
_VAE_TEMPORAL_COMPRESSION = 4
# Wan VAE latent channels.
_NUM_LATENT_CHANNELS = 16
def generation_token_grid(
num_frames: int, latent_h: int, latent_w: int
) -> tuple[int, int, int]:
"""DiT token grid (f, h, w) of a clip of ``num_frames`` pixel frames. Wan VAE: 4k + 1 pixel
frames -> k + 1 latent frames; one latent frame is prepended for the reference image;
the DiT patchifies with patch_size (1, 2, 2), so each spatial side halves."""
clip_latent_t = (num_frames - 1) // _VAE_TEMPORAL_COMPRESSION + 1
return clip_latent_t + 1, latent_h // 2, latent_w // 2
def _generation_latent_shape(
latent_h: int, latent_w: int, num_frames: int
) -> tuple[int, int, int, int]:
"""``[16, latent_t, latent_h, latent_w]`` of the latent the DiT denoises for a clip of
``num_frames`` pixel frames; latent_t comes from generation_token_grid (clip frames plus
the leading reference-image slot)."""
latent_t = generation_token_grid(num_frames, latent_h, latent_w)[0]
return (_NUM_LATENT_CHANNELS, latent_t, latent_h, latent_w)
def get_sampling_sigmas(sampling_steps: int, shift: float) -> np.ndarray:
"""Official Wan sampler grid: ``[sampling_steps]`` fp64 sigmas, uniform on [1, 0) with ``shift`` applied (sigma_0 == 1)."""
sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps]
sigma = shift * sigma / (1 + (shift - 1) * sigma)
return sigma
class _PerClipDenoisingMetadata(msgspec.Struct, frozen=True, kw_only=True):
"""Schedule metadata for one clip of the reference video."""
# Position in the schedule, 0-based.
clip_index: int
# Start frame (inclusive) in the padded reference video.
frame_start_index: int
# End frame (exclusive) in the padded reference video.
frame_end_index: int
# Pixel-frame count, frame_end_index - frame_start_index, zigzag-padded frames included;
# shorter than clip_len only on the last clip.
num_frames: int
# Leading frames taken from the previous clip's output: 0 for clip 0, else
# num_frames_conditioning.
num_frames_conditioning_for_clip: int
def build_schedule(
reference_video_frames: list[np.ndarray],
clip_len: int,
num_frames_conditioning: int,
) -> list[_PerClipDenoisingMetadata]:
"""Prepare denoising metadata for all the clips of the reference video."""
schedule: list[_PerClipDenoisingMetadata] = []
start = 0
clip_index = 0
num_frames_reference_video = len(reference_video_frames)
while True:
if start + num_frames_conditioning >= num_frames_reference_video:
break
num_frames_conditioning_for_clip = 0 if start == 0 else num_frames_conditioning
clip_len_current = clip_len
if num_frames_reference_video - start < clip_len: # last clip shrinks
clip_len_current = num_frames_reference_video - start
schedule.append(
_PerClipDenoisingMetadata(
clip_index=clip_index,
frame_start_index=start,
frame_end_index=start + clip_len_current,
num_frames=clip_len_current,
num_frames_conditioning_for_clip=num_frames_conditioning_for_clip,
)
)
start += clip_len_current - num_frames_conditioning
clip_index += 1
if not schedule:
raise ValueError(
"Wan-Animate-2 reference video produced 0 clips; check "
"clip_len/num_frames_conditioning and that the video decoded frames."
)
return schedule
class _WanAnimate2Inputs(msgspec.Struct, frozen=True, kw_only=True):
"""The per-request inputs read off the Req."""
reference_image_path: str
reference_video_path: str
prompt: str
prompt_ref: str
negative_prompt: str
width: int
height: int
clip_len: int
num_frames_conditioning: int
fps: int
seed: int
num_inference_steps: int
guidance_scale: float
enable_audio: bool
@classmethod
def from_request(cls, batch: Req) -> _WanAnimate2Inputs:
"""Read the Wan-Animate-2 input fields off the Req (all SamplingParam-backed)."""
reference_image_path = validate_and_get_single_string(
batch.image_path, "image_path"
)
reference_video_path = validate_and_get_single_string(
batch.video_path, "video_path"
)
prompt = validate_and_get_single_string(batch.prompt, "prompt")
prompt_ref = validate_and_get_single_string(batch.prompt_ref, "prompt_ref")
# negative_prompt_embeddings is always computed; an unset negative prompt encodes "".
negative_prompt = validate_and_get_single_string(
batch.negative_prompt or "", "negative_prompt"
)
seed = batch.seed
if isinstance(seed, list) and len(seed) != 1:
raise ValueError(
f"Wan-Animate-2 takes exactly one 'seed' per request, got {len(seed)}"
)
if isinstance(seed, list):
seed = seed[0]
return _WanAnimate2Inputs(
reference_image_path=reference_image_path,
reference_video_path=reference_video_path,
prompt=prompt,
prompt_ref=prompt_ref,
negative_prompt=negative_prompt,
width=int(batch.width),
height=int(batch.height),
clip_len=int(batch.clip_len),
num_frames_conditioning=_NUM_FRAMES_CONDITIONING,
fps=int(batch.fps),
seed=int(seed),
num_inference_steps=int(batch.num_inference_steps),
guidance_scale=float(batch.guidance_scale),
enable_audio=bool(batch.enable_audio),
)
class WanAnimate2RequestState(msgspec.Struct, kw_only=True):
"""Clip-independent state, computed once, reused by every clip."""
device: torch.device
sp_size: int
generator: torch.Generator
inputs: _WanAnimate2Inputs
# reference image
letterbox_info: LetterboxInfo
reference_image_height: int
reference_image_width: int
latent_h: int # reference_image_height // 8
latent_w: int # reference_image_width // 8
reference_image_condition: torch.Tensor # [20, 1, latent_h, latent_w] bf16
reference_image_embeddings: torch.Tensor # [1, 257, 1280] bf16
# text
prompt_embeddings: torch.Tensor # [L, 4096] bf16
negative_prompt_embeddings: torch.Tensor # [L, 4096] bf16
prompt_ref_embeddings: torch.Tensor # [L_prompt_ref, 4096] bf16
# reference video, zigzag-padded to the clip schedule
reference_video_frames: list[np.ndarray] # resized [H, W, 3] uint8 RGB frames
schedule: list[_PerClipDenoisingMetadata]
# frame count before padding; the final video is trimmed to it
num_reference_video_frames: int
# Token grid of a full-length clip; every clip's attention layout is padded to it.
full_clip_grid_sizes: tuple[int, int, int]
# Reference-video audio, opt-in per request via enable_audio; None gives a silent mp4.
audio: torch.Tensor | None = None # [1, C, L] fp32 in [-1, 1]
audio_sample_rate: int | None = None
# Assembled by the denoising stage for the output stage, [T, H, W, C] uint8.
decoded_frames: np.ndarray | None = None
# Initial noise for clip 0, drawn by the before-denoising stage so that batch.latents is
# the real denoising input; later clips draw from the generator in schedule order.
clip_0_init_noise: torch.Tensor | None = None
# The one batch.extra entry Wan-Animate-2 uses; later stages read it back through
# request_state_from_batch.
WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY = "wan_animate_2_request_state"
def request_state_from_batch(batch: Req) -> WanAnimate2RequestState:
state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
if not isinstance(state, WanAnimate2RequestState):
raise ValueError(
f"batch.extra[{WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY!r}] is missing: "
"WanAnimate2BeforeDenoisingStage must run before this stage."
)
return state
def _mirror_request_state_into_batch(
batch: Req,
request_state: WanAnimate2RequestState,
*,
scheduler: DPMSolverMultistepScheduler,
flow_shift: float,
) -> None:
"""Populate the standard DenoisingStage input fields from the request state.
The clip loop keeps reading ``request_state``; the embeddings, generator, sigma grid and
timesteps exist for the shared stage verification, warmup and residency bookkeeping.
``batch.latents`` is clip 0's initial noise, drawn here (the seed's first draw) and
consumed by ``build_clip_conditioning`` for clip 0, so the sequence of generator draws is
the same as when the loop drew it: numerics are unchanged."""
inputs = request_state.inputs
batch.prompt_embeds = [request_state.prompt_embeddings]
batch.negative_prompt_embeds = [request_state.negative_prompt_embeddings]
batch.image_embeds = [request_state.reference_image_embeddings]
batch.generator = request_state.generator
sigmas = get_sampling_sigmas(inputs.num_inference_steps, flow_shift)
scheduler.set_timesteps(sigmas=sigmas, device=request_state.device)
batch.timesteps = scheduler.timesteps
batch.sigmas = sigmas.tolist()
batch.latents = _draw_clip_0_init_noise(request_state)
batch.raw_latent_shape = batch.latents.shape
def _draw_clip_0_init_noise(request_state: WanAnimate2RequestState) -> torch.Tensor:
"""Draw clip 0's initial noise from the request generator (its first draw) and keep it on
the request state for build_clip_conditioning; clips 1.. then take the following draws."""
init_noise = torch.randn(
*_generation_latent_shape(
request_state.latent_h,
request_state.latent_w,
request_state.schedule[0].num_frames,
),
dtype=torch.float32,
device=request_state.device,
generator=request_state.generator,
)
request_state.clip_0_init_noise = init_noise
return init_noise
VaeEncoderFn = Callable[[list[torch.Tensor]], list[torch.Tensor]]
ImageEmbedderFn = Callable[[list[torch.Tensor]], torch.Tensor]
def build_clip_conditioning(
request_state: WanAnimate2RequestState,
clip_denoising_metadata: _PerClipDenoisingMetadata,
prev_clip_conditioning_frames: torch.Tensor | None,
*,
vae_encoder: VaeEncoderFn,
image_embedder: ImageEmbedderFn,
) -> WanAnimate2ClipConditioning:
"""Build conditioning for a single clip."""
device = request_state.device
latent_h, latent_w = request_state.latent_h, request_state.latent_w
reference_image_height, reference_image_width = (
request_state.reference_image_height,
request_state.reference_image_width,
)
# gen/noise latent geometry
num_frames = clip_denoising_metadata.num_frames
grid_sizes = generation_token_grid(num_frames, latent_h, latent_w)
latent_t = grid_sizes[0]
# The clip's own latent frames, without the prepended reference-image frame.
clip_latent_t = latent_t - 1
init_noise_shape = _generation_latent_shape(latent_h, latent_w, num_frames)
with (
torch.autocast(device_type=device.type, dtype=torch.bfloat16),
torch.no_grad(),
):
# previous-clip condition over the clip's own latent frames (no reference-image frame)
# [4, clip_latent_t, latent_h, latent_w] fp32
prev_clip_condition_mask = make_conditioning_mask(
clip_latent_t,
latent_h,
latent_w,
clip_denoising_metadata.num_frames_conditioning_for_clip,
device=device,
)
# The clip's pixel video: the given frames first, zeros where the DiT generates.
# Built on ``device`` so the frames never leave the GPU.
prev_clip_condition_pixel_space = torch.zeros(
3,
num_frames,
reference_image_height,
reference_image_width,
device=device,
)
if clip_denoising_metadata.num_frames_conditioning_for_clip > 0:
# prev_clip_conditioning_frames: the previous clip's last
# ``num_frames_conditioning`` decoded frames,
# [C, num_frames_conditioning, reference_image_height, reference_image_width] bf16 in [-1, 1].
if prev_clip_conditioning_frames is None:
raise ValueError(
"For all clips except clip-0 `prev_clip_conditioning_frames` must be not None."
)
prev_clip_condition_pixel_space[
:, : clip_denoising_metadata.num_frames_conditioning_for_clip
] = torch.nn.functional.interpolate(
prev_clip_conditioning_frames[
:, : clip_denoising_metadata.num_frames_conditioning_for_clip
],
size=(reference_image_height, reference_image_width),
mode="bicubic",
)
# [16, clip_latent_t, latent_h, latent_w] fp32
prev_clip_condition = vae_encoder([prev_clip_condition_pixel_space])[0]
# [20, clip_latent_t, latent_h, latent_w] bf16
prev_clip_condition = torch.concat(
[prev_clip_condition_mask, prev_clip_condition]
).to(torch.bfloat16)
# [20, latent_t, latent_h, latent_w] bf16
# recall: latent_t = clip_latent_t + 1 (+1 is for the reference image condition in latent space)
generation_condition = torch.concat(
[request_state.reference_image_condition, prev_clip_condition], dim=1
)
# reference-video slice -> reference_video_latents
# [t, h, w, c] uint8
reference_video_clip_np = np.stack(
request_state.reference_video_frames[
clip_denoising_metadata.frame_start_index : clip_denoising_metadata.frame_end_index
]
)
reference_video_pixels = rearrange(
torch.from_numpy(reference_video_clip_np).to(device), "t h w c -> c t h w"
)
# [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, t, h, w]
reference_video_pixels = (reference_video_pixels.float() / 127.5 - 1).to(
torch.bfloat16
)
# [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32
# Stays fp32 (vae.encode returns .float()); forward_ref cats it with the
# bf16 reference_video_condition under autocast type promotion.
reference_video_latents = vae_encoder([reference_video_pixels])[0]
(
_,
reference_video_latent_t,
reference_video_latent_h,
reference_video_latent_w,
) = reference_video_latents.shape
reference_video_grid_sizes = (
reference_video_latent_t,
reference_video_latent_h // 2,
reference_video_latent_w // 2,
)
# [3, reference_video_height, reference_video_width] bf16
reference_video_frame_0 = reference_video_pixels[:, 0]
reference_video_frame_0_image_embeddings = image_embedder(
[reference_video_frame_0[:, None, :, :]]
).to(torch.bfloat16) # [1, 257, 1280] bf16
# reference_video_condition (forward_ref): mask, then VAE latent of the reference-video slice.
# Every frame of the slice is given, so the latent is reference_video_latents itself;
# the official code encodes the same pixels a second time here.
reference_video_condition_mask = make_conditioning_mask(
reference_video_latent_t,
reference_video_latent_h,
reference_video_latent_w,
num_frames,
device=device,
)
# [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16
reference_video_condition = torch.concat(
[reference_video_condition_mask, reference_video_latents]
).to(torch.bfloat16)
# Drawn from the request's seeded generator; clips are built in schedule order, so
# clip k always gets the k-th draw and (seed, clip_index) identifies the noise. Clip 0's
# draw already happened in the before-denoising stage (it is batch.latents).
if (
clip_denoising_metadata.clip_index == 0
and request_state.clip_0_init_noise is not None
):
init_noise = request_state.clip_0_init_noise
else:
init_noise = torch.randn(
*init_noise_shape,
dtype=torch.float32,
device=device,
generator=request_state.generator,
)
return WanAnimate2ClipConditioning(
reference_video_latents=reference_video_latents,
generation_video_grid_sizes=grid_sizes,
reference_video_frame_0_image_embeddings=reference_video_frame_0_image_embeddings,
reference_video_condition=reference_video_condition,
prompt_ref_embeddings=request_state.prompt_ref_embeddings,
generation_condition=generation_condition,
reference_video_grid_sizes=reference_video_grid_sizes,
full_clip_grid_sizes=request_state.full_clip_grid_sizes,
num_frames=clip_denoising_metadata.num_frames,
init_noise=init_noise,
)
class WanAnimate2BeforeDenoisingStage(TextEncodingStage):
"""Native text encoding plus the model-specific reference and clip conditioning."""
# Text-only stage dedup must not skip fresh image/video conditions or the request RNG.
deduplicated_output_fields = ()
def __init__(
self,
vae: WanAnimate2VaeAdapter,
image_encoder: WanAnimate2ImageEncoderAdapter,
text_encoder: torch.nn.Module,
tokenizer,
pipeline_config: Wan_Animate_2_14B_Config,
scheduler: DPMSolverMultistepScheduler,
) -> None:
super().__init__(text_encoders=[text_encoder], tokenizers=[tokenizer])
self.vae = vae
self.image_encoder = image_encoder
self.pipeline_config = pipeline_config
self._silent_reference_logged = False
self._audio_failure_logged = False
# Shared with the denoising stage, which resets it per clip; used here only to derive
# batch.timesteps on the same sigma grid.
self.scheduler = scheduler
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
name = self._component_stage_name(stage_name)
return super().component_uses(server_args, stage_name) + [
ComponentUse(stage_name=name, component_name="vae"),
ComponentUse(stage_name=name, component_name="image_encoder"),
]
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
device = get_local_torch_device()
sp_size = max(1, int(get_sp_world_size()))
inputs = _WanAnimate2Inputs.from_request(batch)
# One seeded generator per request; clips draw their noise from it in order.
generator = torch.Generator(device=device).manual_seed(inputs.seed)
audio, audio_sample_rate = self._extract_audio(inputs)
request_state = self._build_request_state(
inputs,
device,
sp_size,
generator,
server_args,
audio=audio,
audio_sample_rate=audio_sample_rate,
)
if len(request_state.schedule) > 1:
self.log_info(
"reference video splits into %d clips; each is conditioned in the denoising "
"loop via build_clip_conditioning.",
len(request_state.schedule),
)
batch.extra[WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY] = request_state
_mirror_request_state_into_batch(
batch,
request_state,
scheduler=self.scheduler,
flow_shift=self.pipeline_config.flow_shift,
)
return batch
def _build_request_state(
self,
inputs: _WanAnimate2Inputs,
device: torch.device,
sp_size: int,
generator: torch.Generator,
server_args: ServerArgs,
*,
audio: torch.Tensor | None,
audio_sample_rate: int | None,
) -> WanAnimate2RequestState:
width, height = inputs.width, inputs.height
reference_image = cv2.imread(inputs.reference_image_path)
if reference_image is None:
raise ValueError(
f"Wan-Animate-2: failed to read reference image at {inputs.reference_image_path!r} "
"(cv2.imread returned None: missing file or unsupported/corrupt image)."
)
reference_image = reference_image[..., ::-1] # BGR -> RGB
reference_image, letterbox_info = resize_by_area(
reference_image, width * height, divisor=16
)
# reference video: decode, fps resample, zigzag pad.
frames = read_reference_video_frames(inputs.reference_video_path, inputs.fps)
reference_video_frames = [
resize_by_area(f, width * height, divisor=16)[0] for f in frames
]
num_reference_video_frames = len(frames)
target_len = get_padding_len(num_reference_video_frames, inputs.clip_len)
reference_video_frames = zigzag_padding(reference_video_frames, target_len)
# Keep the official padding and geometry of every needed clip, but do not
# denoise clips whose entire output would be discarded by the final trim.
schedule = [
clip
for clip in build_schedule(
reference_video_frames, inputs.clip_len, inputs.num_frames_conditioning
)
if clip.frame_start_index + clip.num_frames_conditioning_for_clip
< num_reference_video_frames
]
# reference-image encodings (constant across clips)
reference_image_height, reference_image_width = (
reference_image.shape[0],
reference_image.shape[1],
)
latent_h, latent_w = reference_image_height // 8, reference_image_width // 8
reference_image_pixel_values = rearrange(
torch.from_numpy(reference_image).to(device), "h w c -> c 1 h w"
)
# [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, 1, H, W]
reference_image_pixel_values = (
reference_image_pixel_values.float() / 127.5 - 1
).to(torch.bfloat16)
with (
torch.autocast(device_type=device.type, dtype=torch.bfloat16),
torch.no_grad(),
):
# [16, 1, latent_h, latent_w] fp32
reference_image_latents = self.vae_encoder([reference_image_pixel_values])[
0
]
# [4, 1, latent_h, latent_w] fp32
reference_image_mask = make_conditioning_mask(
1, latent_h, latent_w, 1, device=device
)
# [20, 1, latent_h, latent_w] bf16
reference_image_condition = torch.concat(
[reference_image_mask, reference_image_latents]
).to(torch.bfloat16)
reference_image_embeddings = self.image_embedder(
[reference_image_pixel_values]
).to(torch.bfloat16) # [1, 257, 1280] bf16
# Keep batch-one encoder arithmetic while sharing native caching and residency.
prompt_embeddings, prompt_ref_embeddings, negative_prompt_embeddings = [
self.encode_text(prompt, server_args, device=device)[0][0]
for prompt in (inputs.prompt, inputs.prompt_ref, inputs.negative_prompt)
]
return WanAnimate2RequestState(
device=device,
sp_size=sp_size,
generator=generator,
inputs=inputs,
letterbox_info=letterbox_info,
reference_image_height=reference_image_height,
reference_image_width=reference_image_width,
latent_h=latent_h,
latent_w=latent_w,
reference_image_condition=reference_image_condition,
reference_image_embeddings=reference_image_embeddings,
prompt_embeddings=prompt_embeddings,
negative_prompt_embeddings=negative_prompt_embeddings,
prompt_ref_embeddings=prompt_ref_embeddings,
reference_video_frames=reference_video_frames,
schedule=schedule,
num_reference_video_frames=num_reference_video_frames,
full_clip_grid_sizes=generation_token_grid(
inputs.clip_len, latent_h, latent_w
),
audio=audio,
audio_sample_rate=audio_sample_rate,
)
def _extract_audio(
self, inputs: _WanAnimate2Inputs
) -> tuple[torch.Tensor | None, int | None]:
"""Per-request opt-out; any failure yields (None, None), a silent video."""
if not inputs.enable_audio:
return None, None
return extract_reference_video_audio(
inputs.reference_video_path,
log_info=self.log_info,
log_no_audio_track=self._log_silent_reference,
log_warning=self._log_audio_failure,
)
def _log_silent_reference(self, msg: str, *args: Any) -> None:
# A reference video without an audio track is normal input, not a failure:
# one info line per process, then debug.
if self._silent_reference_logged:
self.log_debug(msg, *args)
return
self._silent_reference_logged = True
self.log_info(
msg + " Set enable_audio=false on the request to skip audio extraction.",
*args,
)
def _log_audio_failure(self, msg: str, *args: Any) -> None:
# A track the audio reader cannot decode is the same for every request: warn once
# per process with the remedy, then keep the detail at debug.
if self._audio_failure_logged:
self.log_debug(msg, *args)
return
self._audio_failure_logged = True
self.log_warning(
msg
+ " Set enable_audio=false on the request to skip audio extraction; further "
"audio failures are logged at debug level.",
*args,
)
def vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
with self.use_declared_component(component_name="vae", module=self.vae.vae):
return self.vae.encode(videos)
def image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor:
with self.use_declared_component(
component_name="image_encoder", module=self.image_encoder.model
):
return self.image_encoder.visual(videos)
@@ -0,0 +1,532 @@
# SPDX-License-Identifier: Apache-2.0
"""Wan-Animate-2 denoising and output stages.
WanAnimate2DenoisingStage orchestrates clips around the shared denoising loop: the in-context DiT
takes per-clip conditioning via its ``clip_cond`` kwarg and the clip's reference K/V via
``reference_kv`` (built once per clip with ``build_reference_kv`` and dropped with the clip),
runs cond/uncond CFG as two forwards (the unconditional branch skips block 9 via
``is_unconditional``), resets the scheduler per clip on the official sigma grid, and
VAE-decodes each clip in-loop to feed the next.
WanAnimate2OutputStage then emits the assembled frames.
"""
from __future__ import annotations
from dataclasses import dataclass, field, replace
from typing import TYPE_CHECKING
import numpy as np
import torch
from einops import rearrange
from torch import nn
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import (
get_local_torch_device,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import (
run_cfg_parallel,
)
from sglang.multimodal_gen.runtime.distributed.cfg_policy import (
CFGBranch,
CFGPolicy,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_classifier_free_guidance_world_size,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
WanAnimate2ClipConditioning,
WanAnimate2ReferenceKV,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
DPMSolverMultistepScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
DenoisingContext,
DenoisingStage,
DenoisingStepState,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY,
WanAnimate2RequestState,
build_clip_conditioning,
get_sampling_sigmas,
request_state_from_batch,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
WanAnimate2ImageEncoderAdapter,
WanAnimate2VaeAdapter,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
LetterboxInfo,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
V,
VerificationResult,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
if TYPE_CHECKING:
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
def _is_request_state(value: object) -> bool:
return isinstance(value, WanAnimate2RequestState)
def _is_decoded_frames(value: object) -> bool:
"""``[T, H, W, C]`` uint8 frames assembled by the denoising stage."""
return isinstance(value, np.ndarray) and value.ndim == 4 and value.dtype == np.uint8
@dataclass(kw_only=True)
class WanAnimate2DenoisingContext(DenoisingContext):
clip_condition: WanAnimate2ClipConditioning
reference_image_embeddings: torch.Tensor
cfg_parallel: bool
guidance_scale: float
reference_kv: WanAnimate2ReferenceKV | None = None
context_by_branch: dict[str, torch.Tensor] = field(default_factory=dict)
class WanAnimate2DenoisingStage(DenoisingStage):
"""Wan-Animate-2 in-context denoising over all reference-video clips.
Clip 0 uses the precomputed conditioning; clips > 0 are built from the previous clip's decoded
frames, denoised, decoded in-loop, overlap-trimmed, and assembled into the full video.
"""
def __init__(
self,
*,
transformer: nn.Module,
scheduler: DPMSolverMultistepScheduler,
pipeline: ComposedPipelineBase,
vae: WanAnimate2VaeAdapter,
image_encoder: WanAnimate2ImageEncoderAdapter,
) -> None:
super().__init__(
transformer=transformer, scheduler=scheduler, pipeline=pipeline, vae=vae
)
self.vae: WanAnimate2VaeAdapter = vae
self.clip_decoder = DecodingStage(vae, pipeline=pipeline)
# Clips > 0 are conditioned inside this stage, so it holds the image encoder too.
self.image_encoder = image_encoder
def _vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
with self.use_declared_component(component_name="vae", module=self.vae.vae):
return self.vae.encode(videos)
def _image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor:
with self.use_declared_component(
component_name="image_encoder", module=self.image_encoder.model
):
return self.image_encoder.visual(videos)
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.min_dims(1)])
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(4)])
result.add_check("raw_latent_shape", batch.raw_latent_shape, V.not_none)
result.add_check("sigmas", batch.sigmas, V.list_not_empty)
result.add_check(
"prompt_embeds", batch.prompt_embeds, V.list_of_tensors_dims(2)
)
result.add_check(
"negative_prompt_embeds",
batch.negative_prompt_embeds,
V.list_of_tensors_dims(2),
)
result.add_check("image_embeds", batch.image_embeds, V.list_of_tensors_dims(3))
result.add_check(
"num_inference_steps", batch.num_inference_steps, V.positive_int
)
result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float)
result.add_check("generator", batch.generator, V.generator_or_list_generators)
result.add_check(
WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY,
batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY),
_is_request_state,
)
return result
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
result.add_check(
"decoded_frames",
(
request_state.decoded_frames
if isinstance(request_state, WanAnimate2RequestState)
else None
),
_is_decoded_frames,
)
return result
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
# build_clip_conditioning runs CLIP.visual inside this stage, so image_encoder must stay
# resident (it is CPU-offloaded by default for video tasks).
uses = super().component_uses(server_args, stage_name)
# The VAE adapter runs the VAE in fp32 with autocast off; do not let the manager
# cast it to the configured vae precision on the way in.
uses = [
replace(use, target_dtype=None) if use.component_name == "vae" else use
for use in uses
]
if not any(use.component_name == "image_encoder" for use in uses):
uses.append(
ComponentUse(
stage_name=self._component_stage_name(stage_name),
component_name="image_encoder",
)
)
return uses
@torch.no_grad()
def _denoise_single_clip(
self,
batch: Req,
server_args: ServerArgs,
request_state: WanAnimate2RequestState,
clip_condition: WanAnimate2ClipConditioning,
) -> torch.Tensor:
"""Denoise one clip; return the latents with the reference-image slot dropped.
``batch`` is handed to set_forward_context, which the attention backends read, and
supplies the metrics object the per-step records go to."""
device = get_local_torch_device()
prompt_embeddings = request_state.prompt_embeddings
negative_prompt_embeddings = request_state.negative_prompt_embeddings
reference_image_embeddings = request_state.reference_image_embeddings
# Official sampler grid: sigma_0 is exactly 1.0, so sigmas are set explicitly.
# set_timesteps also resets all solver state for the new clip.
num_inference_steps = request_state.inputs.num_inference_steps
sample_shift = server_args.pipeline_config.flow_shift
self.scheduler.set_timesteps(
sigmas=get_sampling_sigmas(num_inference_steps, sample_shift), device=device
)
guidance_scale = request_state.inputs.guidance_scale
# negative_prompt_embeddings is always computed (an unset prompt encodes ""), so CFG
# depends on the scale alone, unlike Req.validate which also needs a negative prompt.
perform_cfg = guidance_scale > 1
# CFG parallel: cond on cfg_rank 0, uncond on cfg_rank 1, then all-gather. Only
# with --enable-cfg-parallel and initialized model-parallel groups.
cfg_parallel = (
perform_cfg
and bool(server_args.enable_cfg_parallel)
and model_parallel_is_initialized()
)
if cfg_parallel and get_classifier_free_guidance_world_size() != 2:
logger.warning_once(
"CFG parallel enabled but the CFG group world size is "
f"{get_classifier_free_guidance_world_size()} (Wan-Animate-2 expects 2 "
"for its cond/uncond split); using the sequential two-pass CFG path."
)
cfg_parallel = False
branches = [
CFGBranch(
"conditional",
True,
{
"encoder_hidden_states": prompt_embeddings,
"is_unconditional": False,
},
)
]
if perform_cfg:
branches.append(
CFGBranch(
"unconditional",
False,
{
"encoder_hidden_states": negative_prompt_embeddings,
"is_unconditional": True,
},
)
)
cfg_policy = CFGPolicy(branches=branches)
ctx = WanAnimate2DenoisingContext(
scheduler=self.scheduler,
extra_step_kwargs={},
target_dtype=torch.bfloat16,
autocast_enabled=True,
timesteps=self.scheduler.timesteps,
num_inference_steps=num_inference_steps,
num_warmup_steps=0,
image_kwargs={},
pos_cond_kwargs={},
neg_cond_kwargs={},
latents=clip_condition.init_noise,
boundary_timestep=None,
z=None,
reserved_frames_mask=None,
seq_len=None,
guidance=None,
is_warmup=batch.is_warmup,
cfg_policy=cfg_policy,
# DPM-Solver advances once per timestep, irrespective of solver order
extra={"progress_step_interval": 1},
clip_condition=clip_condition,
reference_image_embeddings=reference_image_embeddings,
cfg_parallel=cfg_parallel,
guidance_scale=guidance_scale,
)
# clip trajectories are not part of the assembled-video output contract
self._run_denoising_loop(ctx, batch, server_args, collect_trajectory=False)
return ctx.latents[:, 1:]
def _before_denoising_loop(
self, ctx: WanAnimate2DenoisingContext, batch: Req, server_args: ServerArgs
) -> None:
# activate the DiT before the reference pass, not just the first denoise step
self.begin_declared_component_use(
component_name="transformer", module=self.transformer
)
with set_forward_context(
current_timestep=0, attn_metadata=None, forward_batch=batch
):
ctx.reference_kv = self.transformer.build_reference_kv(
clip_cond=ctx.clip_condition
)
def _prepare_step_state(
self,
ctx: WanAnimate2DenoisingContext,
batch: Req,
server_args: ServerArgs,
step_index: int,
t_host: torch.Tensor,
timesteps_cpu: torch.Tensor,
) -> DenoisingStepState:
return DenoisingStepState(
step_index=step_index,
t_host=t_host,
t_device=ctx.timesteps[step_index],
t_int=int(t_host.item()),
current_model=self.transformer,
current_guidance_scale=ctx.guidance_scale,
attn_metadata=None,
)
def _run_denoising_step(
self,
ctx: WanAnimate2DenoisingContext,
step: DenoisingStepState,
batch: Req,
server_args: ServerArgs,
) -> None:
def predict(branch: CFGBranch) -> torch.Tensor:
with set_forward_context(
current_timestep=step.step_index,
attn_metadata=None,
forward_batch=batch,
):
if branch.name not in ctx.context_by_branch:
ctx.context_by_branch[branch.name] = (
self.transformer.prepare_context(
branch.kwargs["encoder_hidden_states"],
ctx.reference_image_embeddings,
)
)
prediction = self.transformer(
hidden_states=ctx.latents,
timestep=step.t_device.unsqueeze(0),
encoder_hidden_states_image=ctx.reference_image_embeddings,
clip_cond=ctx.clip_condition,
reference_kv=ctx.reference_kv,
projected_context=ctx.context_by_branch[branch.name],
**branch.kwargs,
)
return prediction.contiguous() if ctx.cfg_parallel else prediction
predictions = (
run_cfg_parallel(ctx.cfg_policy, predict)
if ctx.cfg_parallel
else [predict(branch) for branch in ctx.cfg_policy.branches]
)
noise_pred = predictions[0]
if len(predictions) == 2:
noise_pred = predictions[1] + step.current_guidance_scale * (
noise_pred - predictions[1]
)
ctx.latents = ctx.scheduler.step(
noise_pred,
step.t_device,
ctx.latents.unsqueeze(0),
return_dict=False,
)[0].squeeze(0)
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
request_state = request_state_from_batch(batch)
output_frames: list[np.ndarray] = [] # [H, W, C] uint8 frames
# [C, num_frames_conditioning, H, W] bf16 in [-1, 1]
prev_clip_conditioning_frames: torch.Tensor | None = None
# The DiT runs in bf16 autocast like upstream Wan (timestep and norm math in fp32).
with torch.autocast(
device_type=current_platform.device_type,
dtype=torch.bfloat16,
enabled=True,
):
for clip_denoising_metadata in request_state.schedule:
is_clip_0 = clip_denoising_metadata.clip_index == 0
if not is_clip_0 and prev_clip_conditioning_frames is None:
raise ValueError(
"For all clips except clip-0 `prev_clip_conditioning_frames` must be not None."
)
clip_condition = build_clip_conditioning(
request_state,
clip_denoising_metadata,
prev_clip_conditioning_frames,
vae_encoder=self._vae_encoder,
image_embedder=self._image_embedder,
)
latents = self._denoise_single_clip(
batch,
server_args,
request_state,
clip_condition=clip_condition,
)
denoised_latents = latents.to(dtype=torch.float32)
with self.use_declared_component(
component_name="vae", module=self.vae.vae
):
# [1, C, T, H, W] fp32 in [-1, 1]
decoded_frames = self.clip_decoder.decode_raw(
denoised_latents.unsqueeze(0),
server_args,
vae_dtype=torch.float32,
)
# [T, H, W, C] uint8
output_frames_for_clip = (
rearrange(((decoded_frames + 1) * 127.5), "1 c t h w -> t h w c")
.detach()
.to(torch.uint8)
.cpu()
.numpy()
)
# Drop the overlap frames the previous clip already emitted; 0 for clip 0.
output_frames_for_clip = output_frames_for_clip[
clip_denoising_metadata.num_frames_conditioning_for_clip :
]
output_frames.extend(output_frames_for_clip)
# The next clip takes the last ``num_frames_conditioning`` frames.
prev_clip_conditioning_frames = (
decoded_frames[
0, :, -request_state.inputs.num_frames_conditioning :
]
.detach()
.to(torch.bfloat16)
)
# only the independent overlap frames survive into the next clip
del decoded_frames, denoised_latents, latents, clip_condition
if torch.cuda.is_available():
torch.cuda.empty_cache()
# Already-decoded [T, H, W, C] uint8; WanAnimate2OutputStage must not re-decode.
request_state.decoded_frames = self._post_process_frames(
output_frames,
request_state.num_reference_video_frames,
request_state.letterbox_info,
)
return batch
@staticmethod
def _post_process_frames(
frames: list[np.ndarray],
num_output_frames: int,
letterbox_info: LetterboxInfo,
) -> np.ndarray:
"""Stack ``list[np.ndarray]`` to ``[T, H, W, C]`` uint8, trim to ``num_output_frames`` and crop the
``resize_by_area`` letterbox."""
if len(frames) == 0:
raise ValueError(
"WanAnimate2DenoisingStage produced 0 decoded frames to post-process."
)
frames_np = np.stack(frames, axis=0) # [T, H, W, C]
frames_np = frames_np[:num_output_frames]
return letterbox_info.crop(frames_np)
class WanAnimate2OutputStage(PipelineStage):
"""Terminal stage: wrap the already-decoded, assembled frames in an OutputBatch.
Replaces the standard DecodingStage (which would VAE-decode a single-clip latent).
"""
@property
def role_affinity(self) -> RoleType:
# The terminal slot of the standard stage layout; disaggregated deployment itself
# is rejected by Wan_Animate_2_14B_Config.supports_disaggregation.
return RoleType.DECODER
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
result.add_check(
"decoded_frames",
(
request_state.decoded_frames
if isinstance(request_state, WanAnimate2RequestState)
else None
),
_is_decoded_frames,
)
return result
def verify_output(
self, batch: OutputBatch, server_args: ServerArgs
) -> VerificationResult:
result = VerificationResult()
result.add_check("output", batch.output, V.list_not_empty)
return result
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
request_state = request_state_from_batch(batch)
frames = request_state.decoded_frames
if frames is None:
raise ValueError(
"WanAnimate2OutputStage: WanAnimate2DenoisingStage did not assemble decoded frames."
)
# frames: [T, H, W, C] uint8, consumed directly by the save path. Audio, when
# extracted, is [1, C, L] fp32 in [-1, 1] with its sample rate; None gives a silent mp4.
return OutputBatch(
output=[frames],
audio=request_state.audio,
audio_sample_rate=request_state.audio_sample_rate,
metrics=batch.metrics,
)
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
#
# Adapters exposing the official Wan encoder API over the diffusers/transformers models the
# sglang loaders produce: image_encoder.visual takes list[torch.Tensor] of [C, T, H, W] and returns
# [B, 257, 1280]; vae.encode/decode fold in the Wan latent scaling.
from __future__ import annotations
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import CLIPImageProcessor
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
class WanAnimate2ImageEncoderAdapter:
"""Official Wan CLIP API over a ``CLIPVisionModel``: ``visual`` takes ``list[torch.Tensor]`` of ``[C, T, H, W]``
in [-1, 1] and returns ``hidden_states[-2]`` as ``[B, 257, 1280]`` in the model dtype, the pre-final-LayerNorm
slice the Wan-I2V ``postprocess_image`` consumes."""
def __init__(
self,
model: nn.Module,
image_size: int,
image_mean: tuple[float, float, float],
image_std: tuple[float, float, float],
hidden_state_index: int = -2,
device: torch.device | str | None = None,
) -> None:
self.model = model
self.image_size = image_size
self.image_mean = image_mean
self.image_std = image_std
self.hidden_state_index = hidden_state_index
self.device = device
@classmethod
def from_loaded(
cls,
model: nn.Module,
image_processor: CLIPImageProcessor,
device: torch.device | str | None = None,
) -> WanAnimate2ImageEncoderAdapter:
"""Build the adapter with image_size / mean / std from the checkpoint's
``CLIPImageProcessor`` (``crop_size`` is the post-resize crop the model sees)."""
crop_h = int(image_processor.crop_size["height"])
crop_w = int(image_processor.crop_size["width"])
if crop_h != crop_w:
raise ValueError(
"WanAnimate2ImageEncoderAdapter expects a square CLIP crop_size; got "
f"height={crop_h}, width={crop_w}."
)
return cls(
model,
image_size=crop_h,
image_mean=tuple(image_processor.image_mean),
image_std=tuple(image_processor.image_std),
device=device,
)
def _resolve_device(self) -> torch.device:
if self.device is not None:
return torch.device(self.device)
return next(self.model.parameters()).device
@torch.no_grad()
def visual(self, videos: list[torch.Tensor]) -> torch.Tensor:
# The caller holds the component through the residency manager, so the model is
# already where it should be; inputs follow it. Runs in the model's own dtype with
# autocast off: the fused patch-embed conv does not autocast an fp32 weight against
# a bf16 input.
device = self._resolve_device()
model_dtype = next(self.model.parameters()).dtype
size = (self.image_size, self.image_size)
# Each video [C, T, H, W] becomes T images [T, C, image_size, image_size]; the images
# of all videos are stacked into one batch [N, C, image_size, image_size], N = total
# frame count.
pixels = torch.cat(
[
F.interpolate(
video.transpose(0, 1).to(device=device, dtype=model_dtype),
size=size,
mode="bicubic",
align_corners=False,
)
for video in videos
]
)
# [-1, 1] -> [0, 1] then Normalize(mean, std), in the model's own dtype.
mean = torch.tensor(self.image_mean, device=device, dtype=pixels.dtype).view(
1, 3, 1, 1
)
std = torch.tensor(self.image_std, device=device, dtype=pixels.dtype).view(
1, 3, 1, 1
)
pixels = pixels.mul(0.5).add(0.5).sub(mean).div(std)
# Encode; return the hidden_states[-2] slice.
with (
torch.autocast(device_type=device.type, enabled=False),
set_forward_context(current_timestep=0, attn_metadata=None),
):
outputs = self.model(pixel_values=pixels, output_hidden_states=True)
return outputs.hidden_states[self.hidden_state_index]
class WanAnimate2VaeAdapter(nn.Module):
"""Stage-local latent-scaling adapter over the pipeline's native ``AutoencoderKLWan``.
``encode`` takes ``list[torch.Tensor]`` (or ``[B, C, T, H, W]``) and returns ``list[torch.Tensor]`` of
``[16, T', H', W']`` fp32, ``(mode() - latents_mean) / latents_std``; ``decode`` inverts that and returns
``[B, C, T, H, W]`` fp32 in [-1, 1], which is why ``Wan_Animate_2_14B_Config.get_decode_scale_and_shift``
returns ``(1.0, None)``."""
@property
def use_parallel_decode(self) -> bool:
return self.vae.use_parallel_decode
def enable_tiling(self) -> None:
self.vae.enable_tiling()
def __init__(self, vae: nn.Module) -> None:
super().__init__()
if not isinstance(vae, nn.Module):
raise TypeError(
"WanAnimate2VaeAdapter expects an nn.Module AutoencoderKLWan; got "
f"{type(vae).__name__!r}."
)
self.vae = vae
# Per-channel latent scaling (length z_dim), which the sglang AutoencoderKLWan
# copies from its config onto the module. Plain (unregistered) tensors, reshaped
# to broadcast over [B, C, T, H, W].
self._latents_mean = torch.as_tensor(
vae.latents_mean, dtype=torch.float32
).view(1, -1, 1, 1, 1)
self._latents_std = torch.as_tensor(vae.latents_std, dtype=torch.float32).view(
1, -1, 1, 1, 1
)
@staticmethod
def _to_list_of_tensors(
items: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
) -> list[torch.Tensor]:
"""Per-item tensors from either a list or a batched ``[B, ...]`` tensor; the stages
pass lists, the shared decode path passes a batch."""
if isinstance(items, (list, tuple)):
return list(items)
return [items[i] for i in range(items.shape[0])]
def _mean_std_like(self, input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
return (
self._latents_mean.to(device=input.device, dtype=input.dtype),
self._latents_std.to(device=input.device, dtype=input.dtype),
)
@torch.no_grad()
def encode(
self,
videos: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
*args: Any,
**kwargs: Any,
) -> list[torch.Tensor]:
out: list[torch.Tensor] = []
for sample in self._to_list_of_tensors(videos):
# Run fp32 with autocast off: the fused causal-conv path has no dtype guard, so
# an outer autocast(bf16) would cast the conv input against the fp32 bias.
# The sglang AutoencoderKLWan returns the DiagonalGaussianDistribution itself,
# not a diffusers AutoencoderKLOutput.
with torch.autocast(device_type=sample.device.type, enabled=False):
posterior = self.vae.encode(sample.unsqueeze(0).float())
# Deterministic mu (not a sampled draw).
mu = posterior.mode().float()
mean, std = self._mean_std_like(mu)
z = (mu - mean) / std
out.append(z.squeeze(0))
return out
@torch.no_grad()
def decode(
self,
latents: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
*args: Any,
**kwargs: Any,
) -> torch.Tensor:
out: list[torch.Tensor] = []
for sample in self._to_list_of_tensors(latents):
z = sample.unsqueeze(0).float()
mean, std = self._mean_std_like(z)
z = z * std + mean
# fp32, autocast OFF: see ``encode`` above. decode returns a plain tensor.
with torch.autocast(device_type=z.device.type, enabled=False):
decoded = self.vae.decode(z)
out.append(decoded.float().clamp_(-1.0, 1.0).squeeze(0))
# Clips are decoded individually; avoid copying their full pixel tensor.
return out[0].unsqueeze(0) if len(out) == 1 else torch.stack(out, dim=0)
def forward(
self,
z: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
*args: Any,
**kwargs: Any,
) -> torch.Tensor:
return self.decode(z, *args, **kwargs)
@@ -0,0 +1,324 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from the Wan-Animate-2 reference implementation (Apache-2.0):
# https://github.com/Wan-Video/Wan-Animate-2
#
# Keep the original Wan preprocessing numerics; the newer Diffusers modular
# processor is not a numerically interchangeable replacement. The
# zigzag_padding / resize_by_area names are kept for parity with the official helpers;
# make_conditioning_mask is the official get_i2v_mask.
from __future__ import annotations
import math
from copy import deepcopy
from io import BytesIO
from typing import Literal
import cv2
import msgspec
import numpy as np
import torch
from sglang.srt.utils.common import get_video_bytes
__all__ = [
"LetterboxInfo",
"MIN_SINGLE_CLIP_LEN",
"resize_by_area",
"letterbox_resize",
"get_frame_indices",
"get_padding_len",
"single_clip_reference_video_len",
"zigzag_padding",
"make_conditioning_mask",
"read_reference_video_frames",
"write_reference_video",
]
class LetterboxInfo(msgspec.Struct, frozen=True):
"""Where letterbox_resize put the content inside the padded canvas."""
# Axis the padding bars sit on: "width" pads left/right, "height" pads top/bottom.
pad_axis: Literal["width", "height"]
# Bar size in pixels before the content along pad_axis.
padding: int
# Content extent in pixels along pad_axis.
content_len: int
def crop(self, frames: np.ndarray) -> np.ndarray:
"""Cut the bars back out of ``[T, H, W, C]`` frames."""
start, stop = self.padding, self.padding + self.content_len
if self.pad_axis == "width":
return frames[:, :, start:stop, :]
return frames[:, start:stop, :, :]
def resize_by_area(
image: np.ndarray,
target_area: int,
keep_aspect_ratio: bool = True,
divisor: int = 16,
padding_color: tuple[int, int, int] = (0, 0, 0),
) -> tuple[np.ndarray, LetterboxInfo]:
"""Resize ``image`` (``[H, W, C]`` uint8) so its area is <= ``target_area``, then letterbox-pad to that size."""
h, w = image.shape[:2]
aspect_ratio = w / h
if keep_aspect_ratio:
height_target = math.sqrt(target_area / aspect_ratio)
width_target = target_area / height_target
else:
width_target = height_target = math.sqrt(target_area)
width_target, height_target = (
int((width_target // divisor) * divisor),
int((height_target // divisor) * divisor),
)
# Resize (INTER_AREA when shrinking, INTER_LINEAR when enlarging)
# Taken as is from the reference implementation.
interpolation = (
cv2.INTER_AREA if (width_target * height_target < w * h) else cv2.INTER_LINEAR
)
return letterbox_resize(
image,
height=height_target,
width=width_target,
padding_color=padding_color,
interpolation=interpolation,
)
def letterbox_resize(
image: np.ndarray,
height: int = 512,
width: int = 512,
padding_color: tuple[int, int, int] = (0, 0, 0),
interpolation: int = cv2.INTER_LINEAR,
) -> tuple[np.ndarray, LetterboxInfo]:
"""Aspect-preserving resize of ``image`` into a ``[height, width, C]`` uint8 canvas with letterbox padding."""
height_source = image.shape[0]
width_source = image.shape[1]
num_channels_source = image.shape[2]
canvas = np.zeros((height, width, num_channels_source))
if num_channels_source == 1:
canvas[:, :, 0] = padding_color[0]
else:
canvas[:, :, 0] = padding_color[0]
canvas[:, :, 1] = padding_color[1]
canvas[:, :, 2] = padding_color[2]
if (height_source / width_source) > (height / width):
width_target = int(height / height_source * width_source)
img = cv2.resize(image, (width_target, height), interpolation=interpolation)
padding = int((width - width_target) / 2)
if len(img.shape) == 2:
img = img[
:, :, np.newaxis
] # grayscale image: add a num_channels_source dimension
canvas[:, padding : padding + width_target, :] = img
pad_axis = "width"
content_len = width_target
else:
height_target = int(width / width_source * height_source)
img = cv2.resize(image, (width, height_target), interpolation=interpolation)
padding = int((height - height_target) / 2)
if len(img.shape) == 2:
img = img[
:, :, np.newaxis
] # grayscale image: add a num_channels_source dimension
canvas[padding : padding + height_target, :, :] = img
pad_axis = "height"
content_len = height_target
canvas = np.uint8(canvas)
return canvas, LetterboxInfo(
pad_axis=pad_axis, padding=padding, content_len=content_len
)
def get_frame_indices(
num_frames_source: int, fps_source: float, num_frames_target: int, fps_target: float
) -> list[int]:
"""Source frame index (``list[int]``, nearest frame clipped to the source range) for each of the
``num_frames_target`` output frames when resampling from ``fps_source`` to ``fps_target``."""
times = np.arange(0, num_frames_target) / fps_target
frame_indices_source = np.round(times * fps_source).astype(int)
frame_indices_source = np.clip(frame_indices_source, 0, num_frames_source - 1)
return frame_indices_source.tolist()
# Fewest frames the last clip may add beyond its overlap; a multiple of 4 so it also sits
# on the temporal-VAE grid. Taken as is from the reference.
_MIN_LAST_CLIP_NEW_FRAMES = 28
def get_padding_len(
num_frames: int, clip_len: int, num_frames_conditioning: int = 1
) -> int:
"""Reference-video length after zigzag padding so the last clip is long enough and
4k-aligned; taken as is from the reference implementation."""
remaining = (num_frames - num_frames_conditioning) % (
clip_len - num_frames_conditioning
)
# Short leftover: raise it to the minimum. Otherwise: round up to the next multiple
# of 4 (VAE temporal stride); the minimum is itself a multiple of 4.
if remaining < _MIN_LAST_CLIP_NEW_FRAMES:
padding_needed = _MIN_LAST_CLIP_NEW_FRAMES - remaining
else:
padding_needed = 4 - remaining % 4
return num_frames + padding_needed
# get_padding_len extends every reference video to at least this many frames, so a shorter
# clip_len can never hold the whole video in one clip.
MIN_SINGLE_CLIP_LEN = _MIN_LAST_CLIP_NEW_FRAMES + 1
def single_clip_reference_video_len(
clip_len: int, num_frames_conditioning: int = 1
) -> int:
"""Reference-video frame count that get_padding_len extends to exactly ``clip_len``,
so build_schedule yields one full-length clip; used to size the synthetic warmup video."""
num_frames = clip_len - 1
if (
clip_len < MIN_SINGLE_CLIP_LEN
or get_padding_len(num_frames, clip_len, num_frames_conditioning) != clip_len
):
raise ValueError(
f"clip_len={clip_len} cannot hold a reference video in a single clip; "
f"the schedule needs clip_len >= {MIN_SINGLE_CLIP_LEN}"
)
return num_frames
def zigzag_padding(array: list, target_len: int) -> list:
"""Pad ``array`` up to ``target_len`` by bouncing the read index back and forth (deep copies)."""
if target_len < len(array):
raise ValueError(
f"zigzag_padding only extends: target_len {target_len} < len(array) {len(array)}"
)
index = 0
flip = False
target_array = []
while len(target_array) < target_len:
target_array.append(deepcopy(array[index]))
if flip:
index -= 1
else:
index += 1
if index == 0 or index == len(array) - 1:
flip = not flip
return target_array
def make_conditioning_mask(
latent_t: int,
latent_h: int,
latent_w: int,
num_prefix_conditioning_frames: int = 1,
device: torch.device | str = "cuda",
) -> torch.Tensor:
"""Build the ``[4, latent_t, latent_h, latent_w]`` fp32 i2v conditioning mask; ``num_prefix_conditioning_frames`` is the
conditioned prefix length in pixel frames."""
# The VAE maps the first frame to 1 frame in latent space.
# For each subsequent 4 pixel frames it maps them to 1 frame in the latent space.
# Hence, the ``(latent_t - 1) * 4 + 1``.
# VAE: 4k + 1 pixel frames -> k + 1 latent frames
mask = torch.zeros(1, (latent_t - 1) * 4 + 1, latent_h, latent_w, device=device)
mask[:, :num_prefix_conditioning_frames] = 1
mask = torch.concat(
[torch.repeat_interleave(mask[:, 0:1], repeats=4, dim=1), mask[:, 1:]], dim=1
) # repeat the first pixel frame's flag 4x: (k - 1) * 4 + 1 -> 4 * k pixel slots
mask = mask.view(1, mask.shape[1] // 4, 4, latent_h, latent_w)
# (1, 4*(k + 1), latent_h, latent_w) -> (1, (k + 1), 4, latent_h, latent_w)
mask = mask.transpose(1, 2)[0]
# (1, (k + 1), 4, latent_h, latent_w) -> (4, (k + 1), latent_h, latent_w)
return mask
def read_reference_video_frames(path: str, fps_target: int) -> np.ndarray:
"""Decode the reference video resampled to ``fps_target``; returns ``[N, H, W, 3]`` uint8 RGB.
Three readers: decord on CPU as the reference implementation does (its GPU decoder
exists only in CUDA builds); decord2 is the arm64 build of the same API; PyAV is the
fallback when neither is installed.
"""
source = (
BytesIO(get_video_bytes(path))
if path.startswith(("http://", "https://", "data:"))
else path
)
try:
from decord import VideoReader
except ImportError:
try:
from decord2 import VideoReader
except ImportError:
VideoReader = None
if VideoReader is not None:
video_reader = VideoReader(source)
num_frames_source = len(video_reader)
fps_source = video_reader.get_avg_fps()
num_frames_target = int(num_frames_source / fps_source * fps_target)
indices = get_frame_indices(
num_frames_source, fps_source, num_frames_target, fps_target
)
return video_reader.get_batch(indices).asnumpy()
import av
# Decode everything once so num_frames_source matches decord's len() exactly.
with av.open(source) as container:
stream = container.streams.video[0]
fps_source = float(stream.average_rate)
frames_source = [
frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)
]
num_frames_source = len(frames_source)
num_frames_target = int(num_frames_source / fps_source * fps_target)
indices = get_frame_indices(
num_frames_source, fps_source, num_frames_target, fps_target
)
return np.stack([frames_source[i] for i in indices])
def write_reference_video(path: str, frames: np.ndarray, fps: int) -> None:
"""Write ``[N, H, W, 3]`` uint8 RGB frames to ``path`` as an mp4 at ``fps`` through the
runtime's video writer (the one request outputs go through), so the synthetic warmup
video is encoded exactly like a request output and decodes with the readers above."""
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
from sglang.multimodal_gen.runtime.entrypoints.utils import post_process_sample
try:
post_process_sample(frames, DataType.VIDEO, fps, save_file_path=path)
except Exception as e:
raise RuntimeError(
f"the shared video writer (post_process_sample) could not write the "
f"synthetic reference video {path}: {e}"
) from e
def validate_and_get_single_string(value: str | list[str] | None, name: str) -> str:
"""This pipeline produces one video per request, so a batched prompt field,
video path or image path must hold exactly one string."""
if isinstance(value, list):
if len(value) != 1:
raise ValueError(
f"Wan-Animate-2 takes exactly one '{name}' per request, got {len(value)}"
)
value = value[0]
if not isinstance(value, str):
raise ValueError(
f"Wan-Animate-2 requires '{name}' to be a string, got {value!r}"
)
return value
@@ -270,6 +270,18 @@
"ssim_threshold": 0.48,
"psnr_threshold": 13.0,
"mean_abs_diff_threshold": 45.0
},
"wan2_2_animate_2_14b_ref2v": {
"clip_threshold": 0.90,
"ssim_threshold": 0.92,
"psnr_threshold": 24.0,
"mean_abs_diff_threshold": 10.0
},
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
"clip_threshold": 0.90,
"ssim_threshold": 0.92,
"psnr_threshold": 24.0,
"mean_abs_diff_threshold": 10.0
}
},
"default_clip_threshold_image": 0.92,
@@ -39,6 +39,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
SANA_WM_TI2V_CI_sampling_params,
T2I_sampling_params,
T2V_sampling_params,
WAN_2_2_ANIMATE_2_14B_sampling_params,
_make_modelopt_ci_case,
_with_default_num_gpus,
)
@@ -63,6 +64,7 @@ from sglang.multimodal_gen.test.test_utils import (
DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST,
@@ -459,6 +461,23 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
run_models_api_check=False,
run_t2v_input_reference_check=False,
),
# Reference image + driving video -> video. The DiT peaks at 76 GB resident,
# so the 80 GB lane streams it layerwise; this also covers the uncond block-9 skip.
DiffusionTestCase(
"wan2_2_animate_2_14b_ref2v",
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
modality="video",
dit_layerwise_offload=True,
text_encoder_cpu_offload=True,
extras=["--layerwise-offload-components", "transformer"],
),
WAN_2_2_ANIMATE_2_14B_sampling_params,
# The DiT forward takes a per-clip conditioning container and runs forward_ref,
# so the raw-component harness contract does not apply.
run_perf_check=True,
run_component_accuracy_check=False,
),
# flaky
# === Helios T2V ===
# DiffusionTestCase(
@@ -975,6 +994,32 @@ TWO_GPU_CASES = [
model_path=DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
),
),
# GT = the 1-GPU frames renamed. With replicated encoders, serial VAE decode and the
# VAE channels_last_3d layout the loader otherwise applies only for num_gpus == 1,
# CFG-parallel output is bit-identical to the 1-GPU case, so both share one ground truth.
DiffusionTestCase(
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu",
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
modality="video",
cfg_parallel=True,
# On 80 GB cards the resident DiT leaves no room for the full-shape warmup probe; stream it layerwise (bit-identical output).
dit_layerwise_offload=True,
text_encoder_cpu_offload=True,
extras=[
"--layerwise-offload-components",
"transformer",
"--encoder-parallel",
"replicate",
"--vae-config.use-parallel-decode",
"false",
],
env_vars={"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "1"},
),
WAN_2_2_ANIMATE_2_14B_sampling_params,
run_perf_check=True,
run_component_accuracy_check=False,
),
DiffusionTestCase(
"wan2_2_t2v_a14b_2gpu",
DiffusionServerArgs(
@@ -3524,6 +3524,60 @@
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 73.7
},
"wan2_2_animate_2_14b_ref2v": {
"stages_ms": {
"WanAnimate2BeforeDenoisingStage": 3783.78,
"WanAnimate2DenoisingStage": 94604.96,
"WanAnimate2OutputStage": 0.06
},
"denoise_step_ms": {
"0": 25832.83,
"1": 7210.15,
"2": 11187.28,
"3": 8460.7,
"4": 8407.93,
"5": 8409.07,
"6": 8408.07,
"7": 8415.78
},
"expected_e2e_ms": 98402.77,
"expected_load_ms": 62901.15,
"expected_avg_denoise_ms": 10791.48,
"expected_median_denoise_ms": 8412.42,
"load_peak_vram_mb": 1776.0,
"runtime_peak_vram_mb": 36322.0,
"load_peak_allocated_mb": 1737.0,
"runtime_peak_allocated_mb": 25748.0,
"estimated_full_test_time_s": 489.9,
"warmup_peak_vram_mb": 14598.0
},
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
"stages_ms": {
"WanAnimate2BeforeDenoisingStage": 5778.62,
"WanAnimate2DenoisingStage": 58196.97,
"WanAnimate2OutputStage": 0.06
},
"denoise_step_ms": {
"0": 21723.06,
"1": 3125.83,
"2": 4335.41,
"3": 4337.63,
"4": 4347.53,
"5": 4339.76,
"6": 4350.68,
"7": 4280.35
},
"expected_e2e_ms": 63988.59,
"expected_load_ms": 64309.0,
"expected_avg_denoise_ms": 6355.03,
"expected_median_denoise_ms": 4338.7,
"load_peak_vram_mb": 1776.0,
"runtime_peak_vram_mb": 36322.0,
"load_peak_allocated_mb": 1737.0,
"runtime_peak_allocated_mb": 25745.0,
"estimated_full_test_time_s": 404.0,
"warmup_peak_vram_mb": 14598.0
}
}
}
@@ -36,6 +36,10 @@ from sglang.multimodal_gen.registry import (
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
from sglang.multimodal_gen.test.test_utils import (
SGL_TEST_FILES_CI_DATA_REPO,
SGL_TEST_FILES_CI_DATA_REVISION,
)
@dataclass
@@ -885,6 +889,35 @@ TURBOWAN_I2V_sampling_params = DiffusionSamplingParams(
fps=4,
)
# Wan-Animate-2 inputs; they ship in the same ci-data-diffusion commit as the GT,
# so the pinned revision must point at a commit that carries them.
_WAN_2_2_ANIMATE_2_CI_INPUT_BASE = (
"https://raw.githubusercontent.com/"
f"{SGL_TEST_FILES_CI_DATA_REPO}/{SGL_TEST_FILES_CI_DATA_REVISION}/"
"diffusion-ci/inputs/wan_animate_2"
)
WAN_2_2_ANIMATE_2_14B_sampling_params = DiffusionSamplingParams(
prompt=(
"Character appearance description: a smiling young woman with shoulder-length "
"dark hair, a mustard-yellow knit sweater, blue jeans and white sneakers. "
"Background description: a plain light-gray studio backdrop."
),
image_path=f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/reference_short.jpg",
# The model reads `size` as an area budget and emits the 16-aligned canvas fitted to
# the reference aspect; 576x1024 (9:16, like the image) maps onto itself.
output_size="576x1024",
fps=16,
output_format="mp4",
# 24 driving frames at 16 fps pad to one 29-frame clip (get_padding_len),
# so the request denoises a single clip; output length follows the driving video.
extras={
"video_url": f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/driving_video_short.mp4",
"clip_len": 37,
"num_inference_steps": 8,
"seed": 42,
},
)
HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams(
prompt="",
image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png",
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
# publish.
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
SGL_TEST_FILES_CI_DATA_REVISION = "17312c8df37e65bffe75818884bc2a5c93cf3094"
SGL_TEST_FILES_CI_DATA_REVISION = "31d8ad69f2434727aad518f0678fbd57823856fe"
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
# when it's regenerated on its own cadence.
@@ -200,6 +200,9 @@ DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST = (
DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST = (
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
)
# MOVA video generation models
DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST = "OpenMOSS-Team/MOVA-360p"
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,799 @@
# SPDX-License-Identifier: Apache-2.0
"""Unit tests for the Wan-Animate-2 block mask, shape math and per-clip reference K/V ownership.
Single-process and CPU only, in the style of ``test_sp_shard.py`` / ``test_usp_ring_tail_pad.py``:
``_get_block_mask_from_layout`` is called unbound with a stub ``self`` and patched module globals; the
shape helpers are called directly. The parallelism rejections live in ``test_wan_animate_2_config.py``.
"""
from collections import OrderedDict
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.models.dits import wan_animate_2_block
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import (
WanAnimate2Transformer3DModel,
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
WanAnimate2TransformerBlock,
_apply_rope_interleaved,
_InContextLayout,
_make_score_mod,
_sp_scatter_out,
)
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
WanAnimate2ClipConditioning,
WanAnimate2ReferenceKV,
)
from sglang.multimodal_gen.runtime.models.dits.wanvideo import (
WanTransformer3DModel,
WanTransformerBlock,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
WanAnimate2DenoisingStage,
)
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
ensure_distributed_env_defaults,
)
_DIT = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2"
_BLOCK = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block"
_DENOISING = (
"sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages"
".wan_animate_2.denoising"
)
# A short last clip against the full-length clip the mask is built for: this clip is
# (2, 1, 2) = 2 frames x 2 tokens, the reference video (1, 1, 2), the full clip (4, 2, 2) =
# 4 frames x 4 tokens. Every mask-side field (max_*) differs from the clip's own, so a mask
# built from the clip's grid, or attention padding that disagrees with the mask, fails below.
_SHORT_CLIP_LAYOUT = _InContextLayout((2, 1, 2), (1, 1, 2), (4, 2, 2))
def _mask_probe(mask_fn):
def valid(q_idx, kv_idx):
out = mask_fn(
torch.tensor(0),
torch.tensor(0),
torch.tensor(q_idx),
torch.tensor(kv_idx),
)
return bool(out.item())
return valid
def test_block_mask_head_agnostic_and_masks_correctly():
captured = {}
def fake_create_block_mask(mask_fn, *, B, H, Q_LEN, KV_LEN, device, _compile):
captured.update(mask_fn=mask_fn, B=B, H=H, Q_LEN=Q_LEN, KV_LEN=KV_LEN)
return "SENTINEL_BLOCK_MASK"
stub = SimpleNamespace(_block_masks=OrderedDict(), _device=torch.device("cpu"))
layout = _SHORT_CLIP_LAYOUT
with patch(f"{_DIT}.create_block_mask", side_effect=fake_create_block_mask) as cbm:
first = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout)
second = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout)
# Cached: the second call never rebuilds.
assert first == "SENTINEL_BLOCK_MASK"
assert second is first
assert cbm.call_count == 1
# Head/batch-agnostic so SP head-sharding cannot change the mask.
assert captured["B"] is None
assert captured["H"] is None
# Sized from the full clip (16 gen + 12 reference-video tokens), each side ceil'd to 128.
assert captured["Q_LEN"] == 128
assert captured["KV_LEN"] == 256
valid = _mask_probe(captured["mask_fn"])
# Generation queries attend every generation token of the full clip, not the gen pad.
assert valid(0, 0) and valid(5, 15)
assert not valid(5, 16)
# A pad query row attends nothing.
assert not valid(16, 0)
# Frame stride is the full clip's 4 tokens per frame, not this clip's 2: query 5 is in
# generation frame 1 and attends reference-video frame 1 (keys 128..131) only. A mask
# built from the clip's own grid would put query 5 in frame 2 and query 2 in frame 1.
assert valid(5, 128) and valid(5, 131)
assert not valid(5, 132)
assert not valid(2, 128)
# Frame 0 is the reference image; it has no reference-video counterpart.
assert not valid(0, 128)
# Frame 3 attends keys 136..139; 140 is past the 12 reference-video tokens (pad).
assert valid(13, 136) and valid(13, 139)
assert not valid(13, 140)
@pytest.mark.parametrize("cache_kind", ["rope", "mask"])
def test_geometry_cache_bounds_and_rebuilds(cache_kind):
cache = OrderedDict()
stub = SimpleNamespace(_device=torch.device("cpu"))
if cache_kind == "rope":
limit = 4
stub._rope_cache = cache
stub.rotary_emb = SimpleNamespace(
forward_uncached=lambda positions: (positions.cos(), positions.sin())
)
def lookup(size):
return WanAnimate2Transformer3DModel._rope_tables(
stub, (0, size, 1), (0, 2), (0, 2)
)
context = nullcontext()
else:
limit = 2
stub._block_masks = cache
def lookup(size):
layout = _InContextLayout((size, 2, 2), (size - 1, 2, 2), (size, 2, 2))
return WanAnimate2Transformer3DModel._get_block_mask_from_layout(
stub, layout
)
context = patch(
f"{_DIT}.create_block_mask",
side_effect=lambda fn, **kwargs: torch.tensor(
[_mask_probe(fn)(0, i) for i in range(32)]
),
)
with context:
first = lookup(2)
oldest_key = next(iter(cache))
for size in range(3, limit + 2):
lookup(size)
assert lookup(2) is first
lookup(limit + 2)
assert oldest_key in cache
for size in range(limit + 3, 2 * limit + 3):
lookup(size)
assert len(cache) == limit
assert oldest_key not in cache
rebuilt = lookup(2)
torch.testing.assert_close(rebuilt, first, rtol=0, atol=0)
def test_attention_pads_tokens_into_the_slots_the_block_mask_admits():
# The mask is built once per request from the full clip while the attention pads every
# clip's tokens; a frame stride or reference-video offset that drifts between the two
# is a silent attention-pattern bug no shape check catches.
layout = _SHORT_CLIP_LAYOUT
captured = {}
def fake_flex_attention(q, k, v, *, block_mask, score_mod):
captured.update(q=q, k=k, v=v, block_mask=block_mask)
return torch.zeros_like(q)
batch, heads, dim = 1, 2, 8
def tokens(values, scale):
base = torch.tensor(values, dtype=torch.float32).view(1, len(values), 1, 1)
return (base * scale).expand(batch, len(values), heads, dim).clone()
# Token ids 1..4 are generation frame 0 (1, 2) and frame 1 (3, 4); 1, 2 x 1000 are the
# reference-video frame.
q, k, v = (
tokens([1, 2, 3, 4], 1.0),
tokens([1, 2, 3, 4], 10.0),
tokens([1, 2, 3, 4], 100.0),
)
reference_k, reference_v = tokens([1, 2], 1000.0), tokens([1, 2], 10000.0)
stub = SimpleNamespace(log_scale=0.0)
with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
out = WanAnimate2TransformerBlock._in_context_attention(
stub, q, k, v, reference_k, reference_v, "SENTINEL_BLOCK_MASK", layout
)
assert captured["block_mask"] == "SENTINEL_BLOCK_MASK"
assert tuple(out.shape) == (batch, 4, heads, dim)
q_padded = captured["q"].transpose(1, 2) # back to [B, L, N, D]
k_padded = captured["k"].transpose(1, 2)
v_padded = captured["v"].transpose(1, 2)
assert tuple(q_padded.shape) == (
batch,
layout.padded_generation_video_len,
heads,
dim,
)
assert tuple(k_padded.shape) == (batch, layout.padded_kv_len, heads, dim)
def slot(t, i):
return t[0, i, 0, 0].item()
# Generation tokens sit frame by frame at the full clip's 4-token stride.
assert [slot(q_padded, i) for i in range(8)] == [1, 2, 0, 0, 3, 4, 0, 0]
assert [slot(k_padded, i) for i in range(8)] == [10, 20, 0, 0, 30, 40, 0, 0]
assert q_padded[:, 8:].abs().sum() == 0
# Reference-video tokens start at padded_generation_video_len with the same stride.
reference_start = layout.padded_generation_video_len
assert [slot(k_padded, reference_start + i) for i in range(4)] == [1000, 2000, 0, 0]
assert [slot(v_padded, reference_start + i) for i in range(4)] == [
10000,
20000,
0,
0,
]
assert k_padded[:, 8:reference_start].abs().sum() == 0
assert k_padded[:, reference_start + 4 :].abs().sum() == 0
# Every slot that holds a real token is one the mask built from the same layout routes:
# all gen pairs, and frame-1 queries (slots 4, 5) to the reference-video keys.
valid = _mask_probe(layout.block_mask_fn())
gen_slots = [0, 1, 4, 5]
assert all(valid(qi, ki) for qi in gen_slots for ki in gen_slots)
assert all(valid(qi, reference_start + ki) for qi in (4, 5) for ki in (0, 1))
assert not any(valid(qi, reference_start + ki) for qi in (0, 1) for ki in (0, 1))
def test_in_context_attention_hands_flex_a_score_mod_only_when_log_scale_is_nonzero():
# At log_scale 0 the bias is an exact identity, so flex must run without a score_mod;
# otherwise it must get the cached partial (compiled flex guards on its identity).
layout = _SHORT_CLIP_LAYOUT
captured = []
def fake_flex_attention(q, k, v, *, block_mask, score_mod):
captured.append(score_mod)
return torch.zeros_like(q)
gen, reference = torch.zeros(1, 4, 2, 8), torch.zeros(1, 2, 2, 8)
with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
for log_scale in (0.0, -1.3):
WanAnimate2TransformerBlock._in_context_attention(
SimpleNamespace(log_scale=log_scale),
gen,
gen,
gen,
reference,
reference,
"SENTINEL_BLOCK_MASK",
layout,
)
assert captured[0] is None
assert captured[1] is _make_score_mod(layout.max_tokens_per_frame, -1.3)
def test_make_score_mod_is_cached_by_key():
# Compiled flex_attention guards on score_mod identity; a fresh closure per call
# would recompile every clip.
assert _make_score_mod(4, 0.0) is _make_score_mod(4, 0.0)
assert _make_score_mod(4, 0.0) is not _make_score_mod(4, 1.3)
assert _make_score_mod(4, 0.0) is not _make_score_mod(8, 0.0)
def test_score_mod_bias_region_matches_the_official_formula():
# The official wan_animate_2_model.py adds log_scale to keys with index in [hw, 2*hw),
# hw = full-clip tokens per frame (external literal: the distilled checkpoint depends on
# it). In the padded [gen | reference-video] layout those are the keys of generation
# frame 1; the reference-video slice at padded_generation_video_len is not biased.
layout = _SHORT_CLIP_LAYOUT
hw = layout.max_tokens_per_frame # 4
score_mod = _make_score_mod(hw, 5.0)
def biased(kv_idx):
out = score_mod(torch.tensor([1.0]), 0, 0, 0, torch.tensor(kv_idx))
return out.item() == 6.0
probes = list(range(0, 3 * hw)) + [layout.padded_generation_video_len]
assert [kv for kv in probes if biased(kv)] == [4, 5, 6, 7]
# Base weights ship log_scale 0.0: the score_mod must then be an exact identity.
identity = _make_score_mod(hw, 0.0)
score = torch.randn(5)
for kv_idx in (0, hw, 2 * hw, layout.padded_generation_video_len):
assert torch.equal(
identity(score.clone(), 0, 0, 0, torch.tensor(kv_idx)), score
)
class _FakeAllToAll4D:
"""Single-process all_to_all_4D for SP=2: split scatter_dim into (SP, .) and fold
that factor onto gather_dim (and the inverse)."""
SP = 2
def __init__(self):
self.calls = []
def __call__(self, x, scatter_dim, gather_dim):
self.calls.append((scatter_dim, gather_dim))
b, s, h, c = x.shape
sp = self.SP
if (scatter_dim, gather_dim) == (2, 1):
assert h % sp == 0
return (
x.reshape(b, s, sp, h // sp, c)
.permute(0, 2, 1, 3, 4)
.reshape(b, sp * s, h // sp, c)
)
if (scatter_dim, gather_dim) == (1, 2):
assert s % sp == 0
return (
x.reshape(b, sp, s // sp, h, c)
.permute(0, 2, 1, 3, 4)
.reshape(b, s // sp, sp * h, c)
)
raise AssertionError((scatter_dim, gather_dim))
def _reference_pass_stub(captured: list) -> SimpleNamespace:
# Minimal block for forward_ref: identity norm/modulation, q/k/v straight from the stream,
# attn1 records its keyword arguments.
def attn1(q, k, v, **kwargs):
captured.append({"q": q, "k": k, **kwargs})
return q
return SimpleNamespace(
local_num_heads=2,
_modulate=lambda t: tuple(torch.zeros(1, 1, 8) for _ in range(6)),
norm1=lambda x, shift, scale: x,
_qkv=lambda x, n: (
x.reshape(1, -1, n, 4),
x.reshape(1, -1, n, 4) + 1,
x.reshape(1, -1, n, 4) + 2,
),
attn1=attn1,
_cross_and_ffn=lambda hidden, attn, *rest: hidden,
)
def _run_reference_pass(
stub,
*,
seq_len: int,
num_valid: int,
sp_size: int,
key_mask=None,
k_cache=None,
v_cache=None,
cos=None,
sin=None,
):
cos = torch.ones(num_valid, 2) if cos is None else cos
sin = torch.zeros(num_valid, 2) if sin is None else sin
with (
patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size),
patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)),
patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x),
):
return WanAnimate2TransformerBlock.forward_ref(
stub,
hidden_states=torch.arange(seq_len * 8, dtype=torch.float32).reshape(
1, seq_len, 8
),
encoder_hidden_states=torch.zeros(1, 4, 8),
timestep_modulation=torch.zeros(1, 6, 8),
reference_video_rope_cos=cos,
reference_video_rope_sin=sin,
reference_video_num_tokens=num_valid,
index=0,
k_cache={} if k_cache is None else k_cache,
v_cache={} if v_cache is None else v_cache,
reference_video_key_mask=key_mask,
)
@pytest.mark.parametrize("seq_len,sp_size", [(6, 1), (8, 2)])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
def test_reference_cache_reuses_rotated_key_and_preserves_padding(
seq_len, sp_size, dtype
):
captured, keys, values = [], {}, {}
angles = torch.arange(12, dtype=torch.float32).view(6, 2) / 7
stub = _reference_pass_stub(captured)
qkv = stub._qkv
raw_keys = []
def project(x, n):
projected = tuple(t.to(dtype) for t in qkv(x, n))
raw_keys.append(projected[1])
return projected
stub._qkv = project
_run_reference_pass(
stub,
seq_len=seq_len,
num_valid=6,
sp_size=sp_size,
k_cache=keys,
v_cache=values,
cos=angles.cos(),
sin=angles.sin(),
)
raw = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 2, 4)
expected = torch.cat(
[
_apply_rope_interleaved(
(raw[:, :6] + 1).to(dtype), angles.cos(), angles.sin()
),
(raw[:, 6:] + 1).to(dtype),
],
dim=1,
)
torch.testing.assert_close(keys[0], expected, rtol=0, atol=0)
torch.testing.assert_close(values[0], (raw + 2).to(dtype), rtol=0, atol=0)
torch.testing.assert_close(keys[0], captured[0]["k"], rtol=0, atol=0)
assert keys[0].data_ptr() == raw_keys[0].data_ptr()
def test_reference_pass_passes_no_key_mask_when_the_sequence_has_no_padding():
# A dense all-true key mask would route the reference stream through SDPA instead of the
# configured attention backend; with no padding the pass must hand attn1 attn_mask=None and
# RoPE the whole sequence.
captured: list = []
_run_reference_pass(
_reference_pass_stub(captured), seq_len=6, num_valid=6, sp_size=1
)
assert len(captured) == 1
assert captured[0]["attn_mask"] is None
assert captured[0]["q"].shape == (1, 6, 2, 4)
@pytest.mark.parametrize("seq_len,sp_size", [(4, 1), (6, 2)])
def test_generation_reuses_reference_cache_without_rotating_it(seq_len, sp_size):
stub = _reference_pass_stub([])
captured = []
def attend(q, k, v, ref_k, ref_v, *rest):
captured.append((q, k, ref_k, ref_v))
return q
stub._in_context_attention = attend
keys = {0: torch.randn(1, 2, 2, 4)}
values = {0: torch.randn_like(keys[0])}
saved_key = keys[0].clone()
angles = torch.arange(8, dtype=torch.float32).view(4, 2) / 7
hidden = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 8)
with (
patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size),
patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)),
patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x),
):
for _ in range(2):
WanAnimate2TransformerBlock.forward_gen(
stub,
hidden_states=hidden,
encoder_hidden_states=torch.zeros(1, 4, 8),
timestep_modulation=torch.zeros(1, 6, 8),
gen_cos=angles.cos(),
gen_sin=angles.sin(),
block_mask=None,
layout=_SHORT_CLIP_LAYOUT,
index=0,
k_cache=keys,
v_cache=values,
)
raw = hidden.view(1, seq_len, 2, 4)
for q, k, ref_k, ref_v in captured:
for actual, source in ((q, raw), (k, raw + 1)):
expected = torch.cat(
[
_apply_rope_interleaved(source[:, :4], angles.cos(), angles.sin()),
source[:, 4:],
],
dim=1,
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
assert ref_k is keys[0] and ref_v is values[0]
torch.testing.assert_close(keys[0], saved_key, rtol=0, atol=0)
def test_reference_pass_masks_the_sp_tail_padding_with_the_per_clip_mask():
# Gathered under SP the sequence carries tail padding: the pad keys are masked, and a mask
# built once per clip by the caller is used as is.
captured: list = []
_run_reference_pass(
_reference_pass_stub(captured), seq_len=8, num_valid=6, sp_size=2
)
mask = captured[0]["attn_mask"]
assert mask is not None and mask.dtype == torch.bool and tuple(mask.shape) == (1, 8)
assert mask[0, :6].all() and not mask[0, 6:].any()
assert captured[0]["skip_sequence_parallel_override"] is True
per_clip = torch.tensor([[True] * 6 + [False] * 2])
captured.clear()
_run_reference_pass(
_reference_pass_stub(captured),
seq_len=8,
num_valid=6,
sp_size=2,
key_mask=per_clip,
)
assert captured[0]["attn_mask"] is per_clip
def test_gather_then_scatter_round_trips():
# _sp_gather_qkv: [B,S_local,N_tp,C] -> [B,S_full,N_local,C]; _sp_scatter_out inverts it.
fake = _FakeAllToAll4D()
with (
patch(
f"{_BLOCK}._sp_gather_qkv",
side_effect=lambda q, k, v: fake(
torch.cat([q, k, v]), scatter_dim=2, gather_dim=1
).chunk(3),
) as gather,
patch(
f"{_BLOCK}._usp_output_all_to_all",
side_effect=lambda x, head_dim: fake(x, scatter_dim=1, gather_dim=2),
) as scatter,
):
q = torch.arange(1 * 4 * 8 * 16, dtype=torch.float32).reshape(1, 4, 8, 16)
k = q + 1000.0
v = q + 2000.0
gq, gk, gv = wan_animate_2_block._sp_gather_qkv(q, k, v)
assert tuple(gq.shape) == (1, 8, 4, 16)
assert tuple(gk.shape) == (1, 8, 4, 16)
assert tuple(gv.shape) == (1, 8, 4, 16)
back = _sp_scatter_out(gq)
assert tuple(back.shape) == tuple(q.shape)
assert torch.equal(back, q)
gather.assert_called_once_with(q, k, v)
scatter.assert_called_once_with(gq, head_dim=2)
assert fake.calls == [(2, 1), (1, 2)]
@pytest.mark.parametrize(
"full_clip_grid, expected",
[
# 4 frames x 4 tokens: 16 gen + 12 reference-video tokens, one 128 block each.
((4, 2, 2), (4, 16, 12, 128, 128, 256)),
# 4 frames x 100 tokens: 400 -> 512, 300 -> 384.
((4, 10, 10), (100, 400, 300, 512, 384, 896)),
],
)
def test_layout_pads_the_full_clip_geometry_to_128_token_blocks(
full_clip_grid, expected
):
# The reference video has no counterpart for the generation stream's reference-image
# frame, so its token count is one frame short of the generation stream's.
layout = _InContextLayout((3, 2, 2), (2, 2, 2), full_clip_grid)
assert (
layout.max_tokens_per_frame,
layout.max_generation_video_num_tokens,
layout.max_reference_video_num_tokens,
layout.padded_generation_video_len,
layout.padded_reference_video_len,
layout.padded_kv_len,
) == expected
class _KvTaggingTransformer:
"""Fake DiT: ``build_reference_kv`` tags the K/V with the clip's reference-video latents,
``__call__`` records the (clip_cond, reference_kv) pair each forward was handed."""
def __init__(self):
self.context_calls = 0
self.forward_calls: list[
tuple[WanAnimate2ClipConditioning, WanAnimate2ReferenceKV]
] = []
def build_reference_kv(self, *, clip_cond):
latents = clip_cond.reference_video_latents
return WanAnimate2ReferenceKV(k={0: latents}, v={0: latents})
def prepare_context(self, prompt_embeddings, image_embeddings):
self.context_calls += 1
return prompt_embeddings
def __call__(self, *, hidden_states, clip_cond, reference_kv, **_unused):
self.forward_calls.append((clip_cond, reference_kv))
return hidden_states.unsqueeze(0)
class _TwoStepScheduler:
timesteps = torch.tensor([1000.0, 500.0])
order = 1
def set_timesteps(self, *, sigmas, device):
pass
def step(self, noise_pred, t, sample, return_dict):
return (sample,)
def _clip_conditioning(reference_video_latents):
return WanAnimate2ClipConditioning(
reference_video_latents=reference_video_latents,
generation_video_grid_sizes=(2, 1, 1),
reference_video_frame_0_image_embeddings=torch.zeros(1, 1, 4),
reference_video_condition=torch.zeros(20, 1, 2, 2),
prompt_ref_embeddings=torch.zeros(1, 4),
generation_condition=torch.zeros(20, 2, 2, 2),
reference_video_grid_sizes=(1, 1, 1),
full_clip_grid_sizes=(2, 1, 1),
num_frames=4,
init_noise=torch.zeros(16, 2, 2, 2),
)
def _batch_without_perf_recording(*, is_warmup=False) -> SimpleNamespace:
"""The clip loop reads only the perf-recording fields off the batch (and hands it to
set_forward_context); no metrics object means no step records."""
return SimpleNamespace(
metrics=None,
perf_dump_path=None,
is_warmup=is_warmup,
rollout=False,
return_trajectory_latents=False,
)
def _fake_denoising_stage(
transformer, *, events: list[str]
) -> WanAnimate2DenoisingStage:
"""Stand-in for WanAnimate2DenoisingStage in _denoise_single_clip; the residency hooks
record into ``events`` so a test can read the use interval back."""
stage = WanAnimate2DenoisingStage.__new__(WanAnimate2DenoisingStage)
stage.transformer = transformer
stage.scheduler = _TwoStepScheduler()
stage.step_profile = lambda: None
stage._apply_nvtx_gate = lambda is_warmup: False
stage.log_info = lambda *args: None
stage.begin_declared_component_use = lambda *, component_name, module: (
events.append(f"begin:{component_name}")
)
stage._finish_active_component_use = lambda: events.append("finish")
return stage
def _single_clip_request_state(guidance_scale: float) -> SimpleNamespace:
return SimpleNamespace(
prompt_embeddings=torch.zeros(1, 4),
negative_prompt_embeddings=torch.zeros(1, 4),
reference_image_embeddings=torch.zeros(1, 1, 4),
inputs=SimpleNamespace(num_inference_steps=2, guidance_scale=guidance_scale),
)
@pytest.mark.parametrize("is_warmup", [False, True])
@pytest.mark.parametrize("guidance_scale", [1.0, 3.0])
def test_each_clip_is_denoised_with_kv_from_its_own_reference_video(
is_warmup, guidance_scale
):
"""Consecutive clips that share seed and clip index but not the reference video (two
requests on one DiT) must each be denoised with K/V built from their own reference video,
never with K/V left over from the previous one."""
transformer = _KvTaggingTransformer()
events = []
stage = _fake_denoising_stage(transformer, events=events)
stage.step_profile = lambda: events.append("profile")
server_args = SimpleNamespace(
pipeline_config=SimpleNamespace(flow_shift=5.0),
enable_cfg_parallel=False,
disable_autocast=False,
)
request_state = _single_clip_request_state(guidance_scale=guidance_scale)
first = _clip_conditioning(torch.full((16, 1, 2, 2), 1.0))
second = _clip_conditioning(torch.full((16, 1, 2, 2), 2.0))
with patch(
f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu")
):
for clip_cond in (first, second):
WanAnimate2DenoisingStage._denoise_single_clip(
stage,
batch=_batch_without_perf_recording(is_warmup=is_warmup),
server_args=server_args,
request_state=request_state,
clip_condition=clip_cond,
)
# 2 clips x 2 steps; every forward of a clip gets the K/V built from that clip's latents.
branches = 2 if guidance_scale > 1 else 1
assert len(transformer.forward_calls) == 4 * branches
assert transformer.context_calls == 2 * branches
assert events.count("profile") == (0 if is_warmup else 4)
for clip_cond, reference_kv in transformer.forward_calls:
assert reference_kv.k[0] is clip_cond.reference_video_latents
assert (
transformer.forward_calls[0][1]
is not transformer.forward_calls[2 * branches][1]
)
def test_dit_work_of_a_clip_runs_inside_one_declared_transformer_use():
"""The stage declares three component uses, so the residency manager does not activate
the DiT at stage entry; the reference pass and every CFG forward of a clip must run after
the transformer use begins and before it is finished, or an offloaded DiT is never
brought back to the device."""
events: list[str] = []
class _EventTransformer(_KvTaggingTransformer):
def build_reference_kv(self, *, clip_cond):
events.append("reference_kv")
return super().build_reference_kv(clip_cond=clip_cond)
def __call__(self, **kwargs):
events.append("forward")
return super().__call__(**kwargs)
def prepare_context(self, *args):
events.append("context")
return super().prepare_context(*args)
stage = _fake_denoising_stage(_EventTransformer(), events=events)
server_args = SimpleNamespace(
pipeline_config=SimpleNamespace(flow_shift=5.0),
enable_cfg_parallel=False,
disable_autocast=False,
)
with patch(
f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu")
):
WanAnimate2DenoisingStage._denoise_single_clip(
stage,
batch=_batch_without_perf_recording(),
server_args=server_args,
# guidance_scale 3.0: two forwards (cond, uncond) per step.
request_state=_single_clip_request_state(guidance_scale=3.0),
clip_condition=_clip_conditioning(torch.zeros(16, 1, 2, 2)),
)
assert events == [
"begin:transformer",
"reference_kv",
"context",
"forward",
"context",
"forward",
"forward",
"forward",
"finish",
]
def _ensure_single_process_parallel_runtime() -> None:
# The DiT's parallel linear layers need the TP group even on a meta device.
if model_parallel_is_initialized():
return
ensure_distributed_env_defaults()
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
def test_dit_constructs_only_its_own_blocks():
"""Every block the DiT constructs is the in-context block; the parent's block set is not
built and then replaced (40 block objects per construction, allocated on whatever device
the loader initializes on)."""
_ensure_single_process_parallel_runtime()
config = WanAnimate2Config()
config.arch_config.num_layers = 2
constructed: list[type] = []
original_init = WanTransformerBlock.__init__
def recording_init(self, *args, **kwargs):
constructed.append(type(self))
original_init(self, *args, **kwargs)
with (
patch.object(WanTransformerBlock, "__init__", recording_init),
torch.device("meta"),
):
model = WanAnimate2Transformer3DModel(config, hf_config={})
assert [type(block) for block in model.blocks] == [WanAnimate2TransformerBlock] * 2
assert constructed == [WanAnimate2TransformerBlock] * 2
def test_dit_inherits_shared_wan_projections_and_output_head():
assert WanAnimate2TransformerBlock._qkv is WanTransformerBlock._qkv
assert (
WanAnimate2TransformerBlock._cross_and_ffn is WanTransformerBlock._cross_and_ffn
)
assert (
WanAnimate2Transformer3DModel._unpatchify is WanTransformer3DModel._unpatchify
)
@@ -204,6 +204,39 @@ class TestFitAutoResidencyProbe:
assert fitted.width * fitted.height <= 832 * 480
class TestProbeFitTerminates:
def test_pinned_frame_count_ends_the_shrink_ladder(self):
"""A model whose sampling params pin num_frames must not hang the probe fit.
The lightened request comes back at the same size, so the fit must stop
instead of retrying the same shape forever. The request is larger than the
bounded warmup, so the measured-shape floor does not end the ladder first.
"""
from dataclasses import dataclass
from sglang.multimodal_gen.runtime.managers.gpu_worker import (
fit_auto_residency_probe,
)
@dataclass
class _PinnedFrames(SamplingParams):
def __post_init__(self) -> None:
self.num_frames = 37
super().__post_init__()
req = Req(sampling_params=_PinnedFrames(width=576, height=1024))
fitted, estimate, steps = fit_auto_residency_probe(
req,
records=[_record(640, 800, 37, peak_gib=54.0)],
free_bytes=20 << 30,
total_bytes=80 << 30,
server_args=_server_args(),
)
assert steps == 0
assert (fitted.width, fitted.height, fitted.num_frames) == (576, 1024, 37)
assert estimate is not None and estimate > 20 << 30
class TestOutOfMemoryClassification:
def test_allocation_failures_from_libraries_count_as_out_of_memory(self):
from sglang.multimodal_gen.runtime.server_warmup import _is_out_of_memory