[diffusion] model: support Wan-Animate-2 (#42941)
Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Ankit Dhall
Mick Qian
parent
0955169ed2
commit
f1e0472fcc
@@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.models.dits.sana_video import SanaVideoConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.stablediffusion3 import (
|
||||
StableDiffusion3TransformerConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
|
||||
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig
|
||||
|
||||
__all__ = [
|
||||
@@ -34,6 +35,7 @@ __all__ = [
|
||||
"LingBotVideoMoEConfig",
|
||||
"LongLive2VideoConfig",
|
||||
"MiniMaxH3DiTConfig",
|
||||
"WanAnimate2Config",
|
||||
"WanVideoConfig",
|
||||
"Hunyuan3DDiTConfig",
|
||||
"MOVAAudioConfig",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Wan-Animate-2 DiT config. The checkpoint is the Wan2.2-I2V-14B backbone with no
|
||||
# extra parameter tensors, so WanVideoArchConfig is reused with in_channels=36,
|
||||
# image_dim=1280, added_kv_proj_dim=5120 and a param-name remap.
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoArchConfig
|
||||
|
||||
# Checkpoint block params use the official Wan names nested under `blocks.{i}.block.*`.
|
||||
# The loader applies mapping rules in insertion order, re-scanning after each hit, so the
|
||||
# extras below chain into the reused Wan lora (official -> hf) and main (hf -> native) maps.
|
||||
_WAN = WanVideoArchConfig()
|
||||
|
||||
|
||||
_WAN_ANIMATE_2_EXTRA_MAPPING: dict[str, str] = {
|
||||
# Strip the `.block` infix so the reused Wan rules match.
|
||||
r"^blocks\.(\d+)\.block\.(.*)$": r"blocks.\1.\2",
|
||||
# Self-attn / cross-attn qk-norms (not covered by the Wan lora map).
|
||||
r"^blocks\.(\d+)\.self_attn\.norm_q\.(.*)$": r"blocks.\1.norm_q.\2",
|
||||
r"^blocks\.(\d+)\.self_attn\.norm_k\.(.*)$": r"blocks.\1.norm_k.\2",
|
||||
r"^blocks\.(\d+)\.cross_attn\.norm_q\.(.*)$": r"blocks.\1.attn2.norm_q.\2",
|
||||
r"^blocks\.(\d+)\.cross_attn\.norm_k\.(.*)$": r"blocks.\1.attn2.norm_k.\2",
|
||||
# Cross-attn I2V image branch -> WanI2VCrossAttention add_k_proj/add_v_proj/norm_added_k.
|
||||
r"^blocks\.(\d+)\.cross_attn\.k_img\.(.*)$": r"blocks.\1.attn2.add_k_proj.\2",
|
||||
r"^blocks\.(\d+)\.cross_attn\.v_img\.(.*)$": r"blocks.\1.attn2.add_v_proj.\2",
|
||||
r"^blocks\.(\d+)\.cross_attn\.norm_k_img\.(.*)$": r"blocks.\1.attn2.norm_added_k.\2",
|
||||
# The Diffusers checkpoint keeps the official self_attn/cross_attn prefixes but names
|
||||
# the projections the diffusers way (to_q/to_k/to_v/to_out.0, add_*_proj).
|
||||
r"^blocks\.(\d+)\.self_attn\.to_out\.0\.(.*)$": r"blocks.\1.to_out.\2",
|
||||
r"^blocks\.(\d+)\.self_attn\.(to_q|to_k|to_v)\.(.*)$": r"blocks.\1.\2.\3",
|
||||
r"^blocks\.(\d+)\.cross_attn\.to_out\.0\.(.*)$": r"blocks.\1.attn2.to_out.\2",
|
||||
r"^blocks\.(\d+)\.cross_attn\.(to_q|to_k|to_v|add_k_proj|add_v_proj|norm_added_k)\.(.*)$": r"blocks.\1.attn2.\2.\3",
|
||||
# `norm3` is the block's only affine LayerNorm and feeds cross-attn; the reused rules
|
||||
# only know its hf name (`norm2`).
|
||||
r"^blocks\.(\d+)\.norm3\.(.*)$": r"blocks.\1.self_attn_residual_norm.norm.\2",
|
||||
# Per-block 6-way modulation -> scale_shift_table.
|
||||
r"^blocks\.(\d+)\.modulation$": r"blocks.\1.scale_shift_table",
|
||||
# Output head (`head.norm` is param-free; no-op if absent).
|
||||
r"^head\.head\.(.*)$": r"proj_out.\1",
|
||||
r"^head\.modulation$": r"scale_shift_table",
|
||||
r"^head\.norm\.(.*)$": r"norm_out.\1",
|
||||
# Embedders in official naming; hf-named checkpoints are covered by the reused rules.
|
||||
r"^text_embedding\.0\.(.*)$": r"condition_embedder.text_embedder.fc_in.\1",
|
||||
r"^text_embedding\.2\.(.*)$": r"condition_embedder.text_embedder.fc_out.\1",
|
||||
r"^time_embedding\.0\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_in.\1",
|
||||
r"^time_embedding\.2\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_out.\1",
|
||||
r"^time_projection\.1\.(.*)$": r"condition_embedder.time_modulation.linear.\1",
|
||||
r"^img_emb\.proj\.0\.(.*)$": r"condition_embedder.image_embedder.norm1.\1",
|
||||
r"^img_emb\.proj\.1\.(.*)$": r"condition_embedder.image_embedder.ff.fc_in.\1",
|
||||
r"^img_emb\.proj\.3\.(.*)$": r"condition_embedder.image_embedder.ff.fc_out.\1",
|
||||
r"^img_emb\.proj\.4\.(.*)$": r"condition_embedder.image_embedder.norm2.\1",
|
||||
# patch_embedding, self/cross-attn q/k/v/o and ffn.* are covered by the reused Wan maps.
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class WanAnimate2ArchConfig(WanVideoArchConfig):
|
||||
"""Wan2.2-I2V-14B backbone dims (inherited) with the Wan-Animate-2 I/O + remap."""
|
||||
|
||||
# Extras first, then Wan lora (official -> hf), then Wan main (hf -> native).
|
||||
param_names_mapping: dict[str, str] = field(
|
||||
default_factory=lambda: {
|
||||
**_WAN_ANIMATE_2_EXTRA_MAPPING,
|
||||
**_WAN.lora_param_names_mapping,
|
||||
**_WAN.param_names_mapping,
|
||||
}
|
||||
)
|
||||
# Only used for re-serialization, which the inference path does not exercise.
|
||||
reverse_param_names_mapping: dict[str, str] = field(
|
||||
default_factory=lambda: dict(_WAN.reverse_param_names_mapping)
|
||||
)
|
||||
# LoRA adapters use the official Wan names; the loader applies this before
|
||||
# param_names_mapping.
|
||||
lora_param_names_mapping: dict[str, str] = field(
|
||||
default_factory=lambda: dict(_WAN.lora_param_names_mapping)
|
||||
)
|
||||
|
||||
# in_channels: 16 noise + 20 conditioning (4-ch i2v mask + 16-ch cond latent).
|
||||
# image_dim: CLIP (XLM-RoBERTa ViT-H/14) embed dim -> image_embedder.
|
||||
# added_kv_proj_dim -> WanI2VCrossAttention (add_k_proj/add_v_proj/norm_added_k).
|
||||
in_channels: int = 36
|
||||
out_channels: int = 16
|
||||
image_dim: int | None = 1280
|
||||
added_kv_proj_dim: int | None = 5120
|
||||
num_layers: int = 40
|
||||
|
||||
# In-context mechanism (no extra params): reference-video RoPE offsets and the flex-attention
|
||||
# score_mod, consumed by forward_ref / forward_gen. The refer_* / log_scale names
|
||||
# match the official and diffusers transformer configs; do not rename. In those
|
||||
# names "refer" means the reference video (the motion source), not the reference image.
|
||||
refer_offset_t: int = 1
|
||||
refer_offset_h: int = 0
|
||||
refer_offset_w: int = (
|
||||
-1
|
||||
) # <0: resolved to the gen grid width in tokens at runtime (official convention)
|
||||
refer_stride: int = 1
|
||||
# Additive attention-score bias on the keys of generation frame 1 (official name);
|
||||
# 0.0 disables it. Read from transformer/config.json when present; the base weights omit it.
|
||||
log_scale: float = 0.0
|
||||
# The image-embedding cross-attention (added_kv_proj_dim) is always built, as in the
|
||||
# official model; the checkpoint config states the same and nothing else reads this.
|
||||
use_img_emb: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# transformer/config.json uses the official Wan names for the backbone dims;
|
||||
# update_model_arch parks unknown keys in extra_attrs, so translate them here.
|
||||
extras = self.extra_attrs
|
||||
if "num_heads" in extras:
|
||||
self.num_attention_heads = extras.pop("num_heads")
|
||||
if "dim" in extras:
|
||||
self.attention_head_dim = extras.pop("dim") // self.num_attention_heads
|
||||
if "in_dim" in extras:
|
||||
self.in_channels = extras.pop("in_dim")
|
||||
if "out_dim" in extras:
|
||||
self.out_channels = extras.pop("out_dim")
|
||||
if not self.use_img_emb:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 always builds the image-embedding cross-attention; "
|
||||
"a checkpoint with use_img_emb=false is a different model."
|
||||
)
|
||||
self.patch_size = tuple(self.patch_size)
|
||||
super().__post_init__()
|
||||
|
||||
|
||||
@dataclass
|
||||
class WanAnimate2Config(DiTConfig):
|
||||
arch_config: WanAnimate2ArchConfig = field(default_factory=WanAnimate2ArchConfig)
|
||||
prefix: str = "WanAnimate2"
|
||||
@@ -3,11 +3,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen import envs
|
||||
from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig
|
||||
from sglang.multimodal_gen.configs.models.dits import WanVideoConfig
|
||||
from sglang.multimodal_gen.configs.models.dits import WanAnimate2Config, WanVideoConfig
|
||||
from sglang.multimodal_gen.configs.models.encoders import (
|
||||
BaseEncoderOutput,
|
||||
CLIPVisionConfig,
|
||||
@@ -45,6 +47,10 @@ def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tenso
|
||||
return prompt_embeds_tensor
|
||||
|
||||
|
||||
def animate2_postprocess_text(outputs: BaseEncoderOutput, text_inputs) -> torch.Tensor:
|
||||
return outputs.last_hidden_state[0, : text_inputs["num_tokens"]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class WanI2VCommonConfig(PipelineConfig):
|
||||
# for all wan i2v pipelines
|
||||
@@ -301,6 +307,127 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig):
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Wan_Animate_2_14B_Config(WanI2V720PConfig):
|
||||
"""Wan-Animate-2 14B (Wan2.2): reference image + reference video -> video.
|
||||
|
||||
Shares the Wan I2V component bundle (T5, CLIP vision, Wan VAE with the encoder
|
||||
loaded) and swaps in the in-context ``WanAnimate2Config`` DiT; a single expert, so no
|
||||
``boundary_ratio`` switching. The bespoke stages read ``flow_shift`` from here
|
||||
(carrying the audio track is a per-request sampling parameter); everything else below
|
||||
restates the Wan2.2 14B defaults instead of inheriting the Wan2.1 ones from the base.
|
||||
"""
|
||||
|
||||
# reference image + reference video -> video is I2V-shaped.
|
||||
task_type: ModelTaskType = ModelTaskType.I2V
|
||||
|
||||
# In-context animation DiT (in_channels=36 = 16 noise + 20 conditioning).
|
||||
dit_config: DiTConfig = field(default_factory=WanAnimate2Config)
|
||||
|
||||
# Denoising stage. Used when building the sigma grid.
|
||||
flow_shift: float | None = 5.0
|
||||
|
||||
postprocess_text_funcs: tuple[Callable, ...] = field(
|
||||
default_factory=lambda: (animate2_postprocess_text,)
|
||||
)
|
||||
|
||||
def tokenize_prompt(self, prompt, tokenizer, tok_kwargs):
|
||||
text_inputs = super().tokenize_prompt(
|
||||
prompt,
|
||||
tokenizer,
|
||||
{**tok_kwargs, "max_length": self.dit_config.arch_config.text_len},
|
||||
)
|
||||
# Count on the host before TextEncodingStage uploads inputs; no GPU scalar sync.
|
||||
text_inputs["num_tokens"] = int(text_inputs["attention_mask"][0].gt(0).sum())
|
||||
return text_inputs
|
||||
|
||||
# The component manager casts the VAE to this dtype at its first use (the reference
|
||||
# encode in the before-denoising stage), so it is the precision the whole bespoke
|
||||
# VAE path runs at; fp32 like the other Wan2.2 configs changes the output.
|
||||
vae_decode_precision: str = "bf16"
|
||||
|
||||
def get_model_deployment_config(self) -> ModelDeploymentConfig:
|
||||
# The Wan2.2 A14B modes (auto mode may layerwise-offload the 14B DiT) with the
|
||||
# resident threshold the base had: at or above 60 GB free the DiT stays resident.
|
||||
return ModelDeploymentConfig(
|
||||
dit_layerwise_offload_modes=("auto", "memory"),
|
||||
auto_dit_offload_prefetch_size=2,
|
||||
keep_resident_min_available_gb=60,
|
||||
keep_resident_components=("dit",),
|
||||
)
|
||||
|
||||
def get_decode_scale_and_shift(
|
||||
self,
|
||||
device: torch.device | None,
|
||||
dtype: torch.dtype | None,
|
||||
vae: torch.nn.Module | None,
|
||||
) -> tuple[float, None]:
|
||||
"""``(1.0, None)``: ``WanAnimate2VaeAdapter.decode`` applies the latent scaling itself,
|
||||
so the DecodingStage must not de-normalize again."""
|
||||
return 1.0, None
|
||||
|
||||
def supports_disaggregation(self) -> bool:
|
||||
# The stages share one WanAnimate2RequestState object (GPU tensors, a torch.Generator,
|
||||
# the clip schedule) through batch.extra. The disaggregation transport forwards only
|
||||
# JSON scalars and tensor pytrees (dict/list/tuple) from batch.extra and never the
|
||||
# generator (disaggregation/extra_tensors.py, scheduler_mixin._EXCLUDE_FIELDS), so a
|
||||
# split-role deployment would silently drop the state and fail at the first request.
|
||||
return False
|
||||
|
||||
def validate_server_args(self, server_args: Any) -> None:
|
||||
if server_args.use_fsdp_inference:
|
||||
raise NotImplementedError(
|
||||
"Wan-Animate-2 does not support --use-fsdp-inference; use "
|
||||
"--dit-layerwise-offload or --tp-size instead."
|
||||
)
|
||||
# Ring splits the K/V across ranks; the in-context block mask indexes the full
|
||||
# [generation | reference-video] K/V on one rank.
|
||||
if server_args.ring_degree > 1:
|
||||
raise NotImplementedError(
|
||||
"Wan-Animate-2 does not support ring sequence parallelism "
|
||||
f"(--ring-degree {server_args.ring_degree}); use --ulysses-degree N "
|
||||
"with --ring-degree 1."
|
||||
)
|
||||
num_heads = self.dit_config.num_attention_heads
|
||||
if num_heads % server_args.tp_size != 0:
|
||||
raise ValueError(
|
||||
f"Wan-Animate-2 tensor parallelism needs the {num_heads} attention "
|
||||
f"heads divisible by --tp-size {server_args.tp_size}."
|
||||
)
|
||||
local_num_heads = num_heads // server_args.tp_size
|
||||
if local_num_heads % server_args.ulysses_degree != 0:
|
||||
raise ValueError(
|
||||
f"Wan-Animate-2 Ulysses SP needs the TP-local head count ({local_num_heads}) "
|
||||
f"divisible by --ulysses-degree {server_args.ulysses_degree}."
|
||||
)
|
||||
# The in-context self-attention runs on torch flex_attention whatever backend is
|
||||
# selected; only the cross-attention and the reference pass would follow the flag.
|
||||
requested_dit_backend = server_args.requested_component_attention_backend(
|
||||
"transformer"
|
||||
)
|
||||
if (
|
||||
server_args.is_arg_explicitly_set("attention_backend")
|
||||
and server_args.attention_backend is not None
|
||||
) or requested_dit_backend is not None:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 does not support selecting the DiT attention backend "
|
||||
f"(--attention-backend {server_args.attention_backend} / "
|
||||
f"--component-attention-backends transformer={requested_dit_backend}): "
|
||||
"its in-context self-attention runs on torch flex_attention only. Drop "
|
||||
"the flag; the text encoder still takes "
|
||||
"--component-attention-backends text_encoder=<backend>."
|
||||
)
|
||||
# The parent block applies the offline Q/K rotation inside its own attention
|
||||
# path; forward_ref / forward_gen never do, so a rotated checkpoint would load
|
||||
# and produce wrong output without a message.
|
||||
if envs.SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 does not support SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION "
|
||||
"(offline Q/K rotation); unset it for this model."
|
||||
)
|
||||
super().validate_server_args(server_args)
|
||||
|
||||
|
||||
# =============================================
|
||||
# ============= Causal Self-Forcing =============
|
||||
# =============================================
|
||||
@@ -322,6 +449,7 @@ def register():
|
||||
Wan2_2_I2V_A14B_SamplingParam,
|
||||
Wan2_2_T2V_A14B_SamplingParam,
|
||||
Wan2_2_TI2V_5B_SamplingParam,
|
||||
Wan_Animate_2_14B_SamplingParam,
|
||||
WanI2V_14B_480P_SamplingParam,
|
||||
WanI2V_14B_720P_SamplingParam,
|
||||
WanT2V_1_3B_SamplingParams,
|
||||
@@ -423,3 +551,20 @@ def register():
|
||||
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
|
||||
],
|
||||
)
|
||||
register_configs(
|
||||
sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
|
||||
pipeline_config_cls=Wan_Animate_2_14B_Config,
|
||||
hf_model_paths=[
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
|
||||
],
|
||||
model_detectors=[
|
||||
# The registry calls this with the path and with model_index's lower-cased
|
||||
# _class_name. Require the "-2-"/"_2" token so a v1 Wan2.2-Animate-14B
|
||||
# checkpoint (a different model / pipeline) does not mis-route here.
|
||||
lambda p: (
|
||||
"wan_animate_2" in p.lower()
|
||||
or "wan2.2-animate-2" in p.lower()
|
||||
or p.lower() == "wananimate2pipeline"
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
||||
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _wan_1_3b_coefficients(p: TeaCacheParams) -> list[float]:
|
||||
@@ -267,6 +274,160 @@ class Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams):
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Wan_Animate_2_14B_SamplingParam(Wan2_2_Base_SamplingParams):
|
||||
"""Sampling defaults for Wan-Animate-2 14B."""
|
||||
|
||||
# Single expert: no guidance_scale_2 / boundary switching.
|
||||
guidance_scale: float = 3.0
|
||||
num_inference_steps: int = 40 # official README demo default
|
||||
|
||||
# The reference video is resampled to this rate and the output MP4 is written at it.
|
||||
fps: int = 16
|
||||
# Carry the reference video's audio track on the output, as the official pipeline does:
|
||||
# the track is decoded in-process and the shared MP4 writer encodes it. False gives a
|
||||
# silent video. A track that cannot be decoded is logged and the output stays silent.
|
||||
enable_audio: bool = True
|
||||
|
||||
# Per-clip denoising length, 4k+1 (VAE stride 4); larger = better temporal coherence
|
||||
# but more VRAM. Output length follows the reference video, not this.
|
||||
clip_len: int = 37
|
||||
# Text for the reference-video branch (prompt_ref_embeds). The official demo conditions it on a
|
||||
# fixed description of the reference video, not on the character caption.
|
||||
prompt_ref: str = "人物动作的参考视频"
|
||||
|
||||
# The reference image comes via the base image_path and the reference video via the
|
||||
# base video_path; neither needs a model-specific field.
|
||||
|
||||
# (width, height) pairs; upstream Wan-Animate default is 640x800.
|
||||
supported_resolutions: list[tuple[int, int]] | None = field(
|
||||
default_factory=lambda: [
|
||||
(640, 800), # upstream Wan-Animate default
|
||||
(720, 1280), # 9:16
|
||||
]
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# The in-context denoising loop runs every block at every step; the shared DiT's
|
||||
# cache heuristics are never consulted.
|
||||
if self.enable_teacache or self.enable_spectrum:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 does not support enable_teacache or enable_spectrum."
|
||||
)
|
||||
requested_clip_len = self.clip_len
|
||||
self.clip_len = _round_clip_len(self.clip_len)
|
||||
if self.clip_len < MIN_CLIP_LEN:
|
||||
raise ValueError(
|
||||
f"clip_len={requested_clip_len!r} is not supported: a clip is 4k+1 frames "
|
||||
f"with k >= 1, so clip_len must be an int >= {MIN_CLIP_LEN}"
|
||||
)
|
||||
if self.clip_len != requested_clip_len:
|
||||
logger.info(
|
||||
"clip_len=%s is not 4k+1; using clip_len=%s",
|
||||
requested_clip_len,
|
||||
self.clip_len,
|
||||
)
|
||||
# The output length follows the reference video and the stages read clip_len, so
|
||||
# `num_frames` is not a length control here; the multi-GPU frame alignment must
|
||||
# leave it alone, and it must stay writable so the warmup probe can shrink.
|
||||
self.adjust_frames = False
|
||||
super().__post_init__()
|
||||
|
||||
@classmethod
|
||||
def video_request_extra_fields(cls) -> frozenset[str]:
|
||||
# Settable over the /v1/videos API: the per-clip length, the reference-branch
|
||||
# text (the official gradio exposes prompt_ref as well) and the audio track. The
|
||||
# reference video is the base video_path field, so it needs no entry here.
|
||||
return frozenset({"clip_len", "prompt_ref", "enable_audio"})
|
||||
|
||||
def prepare_synthetic_warmup_request_for_queue(
|
||||
self, req: Any, server_args: Any
|
||||
) -> None:
|
||||
"""Give the synthetic warmup request the driving video the generic builder does not
|
||||
supply, sized so the request denoises exactly one clip."""
|
||||
del server_args
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
|
||||
single_clip_reference_video_len,
|
||||
write_reference_video,
|
||||
)
|
||||
|
||||
image_path = req.image_path
|
||||
if isinstance(image_path, list):
|
||||
image_path = image_path[0] if image_path else None
|
||||
if not isinstance(image_path, str) or not image_path:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 synthetic warmup requires the warmup reference image"
|
||||
)
|
||||
|
||||
# The builder sizes warmup by num_frames (the class default or its bounded cap), but
|
||||
# the stage reads clip_len and the compiled attention is specialized per clip
|
||||
# geometry, so warm up at the production clip length unless more frames were asked.
|
||||
clip_len = _round_clip_len(max(int(req.num_frames), self.clip_len))
|
||||
num_video_frames = single_clip_reference_video_len(clip_len)
|
||||
fps = int(req.fps)
|
||||
|
||||
# Same directory and lifetime as the warmup image; the name carries the frame count
|
||||
# because every warmup request is built before the first one runs.
|
||||
video_path = os.path.join(
|
||||
os.path.dirname(image_path),
|
||||
f"warmup_driving_video_{num_video_frames}f_{fps}fps.mp4",
|
||||
)
|
||||
write_reference_video(
|
||||
video_path,
|
||||
_synthetic_driving_video_frames(num_video_frames),
|
||||
fps=fps,
|
||||
)
|
||||
|
||||
req.video_path = video_path
|
||||
req.clip_len = clip_len
|
||||
req.num_frames = clip_len
|
||||
# The synthetic driving video has no audio track.
|
||||
req.enable_audio = False
|
||||
|
||||
def _adjust_visual_fields(self, server_args: Any, pipeline_config: Any) -> None:
|
||||
super()._adjust_visual_fields(server_args, pipeline_config)
|
||||
if self.num_frames != type(self).num_frames:
|
||||
logger.info(
|
||||
"Wan-Animate-2 ignores num_frames=%s (the /v1/videos `seconds`): the output "
|
||||
"length follows the reference video; clip_len=%s is the per-clip chunk",
|
||||
self.num_frames,
|
||||
self.clip_len,
|
||||
)
|
||||
|
||||
|
||||
# Smallest clip: one conditioning frame plus one VAE temporal stride (4k+1, k >= 1).
|
||||
MIN_CLIP_LEN = 5
|
||||
|
||||
|
||||
def _round_clip_len(clip_len: int) -> int:
|
||||
"""Clips are 4k+1 pixel frames (Wan VAE temporal stride 4). Other values map to the
|
||||
4k+1 with the same k: 4k rounds up by one frame, 4k+2 and 4k+3 round down."""
|
||||
if clip_len % 4 != 1:
|
||||
clip_len = clip_len // 4 * 4 + 1
|
||||
return clip_len
|
||||
|
||||
|
||||
# Pixel size of the synthetic warmup driving video; the stage rescales every frame to the
|
||||
# request area anyway, so this only has to be a size the shared video writer takes as is
|
||||
# and cheap to encode.
|
||||
_WARMUP_DRIVING_VIDEO_SIZE = 64
|
||||
|
||||
|
||||
def _synthetic_driving_video_frames(num_frames: int) -> np.ndarray:
|
||||
"""``[num_frames, 64, 64, 3]`` uint8 frames: a gradient that drifts one row per frame,
|
||||
so consecutive frames differ like real motion."""
|
||||
size = _WARMUP_DRIVING_VIDEO_SIZE
|
||||
ramp = np.linspace(0, 255, size, dtype=np.float32)
|
||||
rows = np.broadcast_to(ramp[:, None], (size, size))
|
||||
cols = np.broadcast_to(ramp[None, :], (size, size))
|
||||
frames = np.empty((num_frames, size, size, 3), dtype=np.uint8)
|
||||
for index in range(num_frames):
|
||||
frames[index, ..., 0] = np.roll(rows, index, axis=0)
|
||||
frames[index, ..., 1] = cols
|
||||
frames[index, ..., 2] = 127
|
||||
return frames
|
||||
|
||||
|
||||
@dataclass
|
||||
class Turbo_Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams):
|
||||
guidance_scale: float = 3.5 # high_noise
|
||||
|
||||
@@ -710,6 +710,9 @@ class WeightOverrideComponentLoader(ComponentLoader):
|
||||
"""Base for loaders that consume an exact weights-only override."""
|
||||
|
||||
ignored_checkpoint_prefixes: tuple[str, ...] = ()
|
||||
# True: a checkpoint key that maps to no model parameter fails the load instead
|
||||
# of being logged and skipped.
|
||||
strict_checkpoint_keys: bool = False
|
||||
|
||||
def load_state_dict_model(
|
||||
self,
|
||||
@@ -738,7 +741,7 @@ class WeightOverrideComponentLoader(ComponentLoader):
|
||||
fsdp_inference=server_args.should_use_fsdp_for_component(component_name),
|
||||
param_dtype=dtype,
|
||||
reduce_dtype=torch.float32,
|
||||
strict=False,
|
||||
strict=self.strict_checkpoint_keys,
|
||||
weight_load_plan=weight_load_plan,
|
||||
checkpoint_key_filter=checkpoint_key_filter,
|
||||
weights_iterator=weights_iterator,
|
||||
|
||||
@@ -214,22 +214,28 @@ def fit_auto_residency_probe(
|
||||
floor_units = min((record.workload_units() for record in records), default=0)
|
||||
fitted, steps = req, 0
|
||||
while True:
|
||||
units = (
|
||||
max(1, int(fitted.width or 1))
|
||||
* max(1, int(fitted.height or 1))
|
||||
* max(1, int(fitted.num_frames or 1))
|
||||
)
|
||||
units = _probe_workload_units(fitted)
|
||||
estimate = estimate_default_workload_peak_bytes(
|
||||
records=records, target_units=units
|
||||
)
|
||||
if estimate is None or estimate <= budget or units <= floor_units:
|
||||
return fitted, estimate, steps
|
||||
lighter = lighten_warmup_req(server_args, fitted)
|
||||
if lighter is None:
|
||||
# Sampling params that pin the frame count (Wan-Animate-2 mirrors clip_len
|
||||
# in __post_init__) hand back an equal-size req; treat that as the floor.
|
||||
if lighter is None or _probe_workload_units(lighter) >= units:
|
||||
return fitted, estimate, steps
|
||||
fitted, steps = lighter, steps + 1
|
||||
|
||||
|
||||
def _probe_workload_units(req: Req) -> int:
|
||||
return (
|
||||
max(1, int(req.width or 1))
|
||||
* max(1, int(req.height or 1))
|
||||
* max(1, int(req.num_frames or 1))
|
||||
)
|
||||
|
||||
|
||||
class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
"""
|
||||
A worker that executes the model on a single GPU.
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Wan-Animate-2 DiT. Subclasses WanTransformer3DModel so TP / Ulysses SP shard it
|
||||
# unchanged; only the per-block self-attention is swapped for the in-context mechanism
|
||||
# (build_reference_kv runs forward_ref once per clip and returns each block's rotated reference-video
|
||||
# K/V, forward_gen attends over [gen tokens, then that reference-video K/V]). No extra parameters.
|
||||
# Names kept from the reference implementation: the forward_ref / forward_gen methods and the
|
||||
# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image.
|
||||
from collections import OrderedDict
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn.attention.flex_attention import BlockMask, create_block_mask
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
|
||||
WanAnimate2Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_sp_world_size
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
|
||||
gather_seq,
|
||||
shard_seq,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
|
||||
WanAnimate2TransformerBlock,
|
||||
_InContextLayout,
|
||||
_reference_video_key_mask,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
|
||||
WanAnimate2ClipConditioning,
|
||||
WanAnimate2ReferenceKV,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformer3DModel
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
|
||||
class WanAnimate2Transformer3DModel(WanTransformer3DModel):
|
||||
"""Wan-Animate-2 DiT (in-context variant) on the Wan2.2-I2V-14B backbone.
|
||||
|
||||
Attention goes through USPAttention, so callers must wrap the forward in
|
||||
``set_forward_context(...)``.
|
||||
"""
|
||||
|
||||
# Registry alias: the diffusers class name of this transformer.
|
||||
_aliases = ["WanAnimate2Transformer3DModel"]
|
||||
|
||||
param_names_mapping = WanAnimate2Config().param_names_mapping
|
||||
reverse_param_names_mapping = WanAnimate2Config().reverse_param_names_mapping
|
||||
lora_param_names_mapping = WanAnimate2Config().lora_param_names_mapping
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WanAnimate2Config,
|
||||
hf_config: dict[str, Any] | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
config=config, hf_config=hf_config or {}, quant_config=quant_config
|
||||
)
|
||||
|
||||
# RoPE placement of the reference-video tokens relative to the gen grid; plain
|
||||
# ints from the config, not parameters, so the Wan state dict is unchanged.
|
||||
self.refer_offset_t = config.refer_offset_t
|
||||
self.refer_offset_h = config.refer_offset_h
|
||||
# <0: place reference-video tokens right of the gen grid (w offset = gen W in tokens).
|
||||
self.refer_offset_w = config.refer_offset_w
|
||||
self.refer_stride = config.refer_stride
|
||||
# Prompt embeddings are padded/truncated to this many tokens.
|
||||
self.max_text_len = config.text_len
|
||||
|
||||
self._block_masks: OrderedDict[tuple[int, int, int, str], BlockMask] = (
|
||||
OrderedDict()
|
||||
)
|
||||
# RoPE tables keyed by the (f, h, w) position ranges and device; see _rope_tables.
|
||||
self._rope_cache: OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]] = (
|
||||
OrderedDict()
|
||||
)
|
||||
|
||||
def _make_block(
|
||||
self,
|
||||
*,
|
||||
index: int,
|
||||
config: WanAnimate2Config,
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> nn.Module:
|
||||
# Same constructor args as the Wan block and no new params, so the Wan checkpoint
|
||||
# mapping and TP sharding apply unchanged.
|
||||
return WanAnimate2TransformerBlock(
|
||||
config.num_attention_heads * config.attention_head_dim,
|
||||
config.ffn_dim,
|
||||
config.num_attention_heads,
|
||||
config.qk_norm,
|
||||
config.cross_attn_norm,
|
||||
config.eps,
|
||||
config.added_kv_proj_dim,
|
||||
self._supported_attention_backends
|
||||
| {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
|
||||
prefix=f"blocks.{index}",
|
||||
attention_type=config.attention_type,
|
||||
sla_topk=config.sla_topk,
|
||||
quant_config=quant_config,
|
||||
log_scale=config.log_scale,
|
||||
)
|
||||
|
||||
# Device and dtype of the resident weights; the patch embedding is never offloaded.
|
||||
@property
|
||||
def _device(self) -> torch.device:
|
||||
return self.patch_embedding.proj.weight.device
|
||||
|
||||
@property
|
||||
def _dtype(self) -> torch.dtype:
|
||||
return self.patch_embedding.proj.weight.dtype
|
||||
|
||||
def _rope_tables(
|
||||
self,
|
||||
f_range: tuple[int, int, int], # (start, length, stride)
|
||||
h_range: tuple[int, int], # (start, length)
|
||||
w_range: tuple[int, int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""RoPE ``(cos, sin)``, each ``[f*h*w, D//2]`` fp32, for the full grid.
|
||||
|
||||
Row i holds the position of token i, flattened in (f, h, w) row-major order
|
||||
like the patch embedding output.
|
||||
|
||||
The table is cached per ranges."""
|
||||
key = (f_range, h_range, w_range, str(self._device))
|
||||
if key in self._rope_cache:
|
||||
self._rope_cache.move_to_end(key)
|
||||
return self._rope_cache[key]
|
||||
# Bound retained GPU tables; evict before allocating the next geometry.
|
||||
if len(self._rope_cache) >= 4:
|
||||
self._rope_cache.popitem(last=False)
|
||||
f_start, f_len, f_stride = f_range
|
||||
f_idx = torch.arange(f_len, device=self._device) * f_stride + f_start
|
||||
h_idx = torch.arange(h_range[1], device=self._device) + h_range[0]
|
||||
w_idx = torch.arange(w_range[1], device=self._device) + w_range[0]
|
||||
# Positions are built explicitly (not via the rank-sharding forward_from_grid) because
|
||||
# the in-context blocks all-to-all-gather the full sequence before applying RoPE.
|
||||
ff, hh, ww = torch.meshgrid(f_idx, h_idx, w_idx, indexing="ij")
|
||||
positions = torch.stack([ff.reshape(-1), hh.reshape(-1), ww.reshape(-1)], dim=1)
|
||||
cos, sin = self.rotary_emb.forward_uncached(positions.long())
|
||||
tables = (cos.float(), sin.float())
|
||||
self._rope_cache[key] = tables
|
||||
return tables
|
||||
|
||||
def _generation_video_rope(
|
||||
self, f: int, h: int, w: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""RoPE tables for the generated video's ``f x h x w`` token grid; positions start at 0."""
|
||||
return self._rope_tables((0, f, 1), (0, h), (0, w))
|
||||
|
||||
def _reference_video_rope(
|
||||
self, f: int, h: int, w: int, generation_video_w: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""RoPE tables for the reference video's ``f x h x w`` token grid, shifted by the
|
||||
``refer_*`` offsets. When ``refer_offset_w`` is -1, the reference-video grid is placed
|
||||
directly to the right of the generated grid (width offset = ``generation_video_w``)."""
|
||||
offset_w = (
|
||||
generation_video_w if self.refer_offset_w == -1 else self.refer_offset_w
|
||||
)
|
||||
return self._rope_tables(
|
||||
(self.refer_offset_t, f, self.refer_stride),
|
||||
(self.refer_offset_h, h),
|
||||
(offset_w, w),
|
||||
)
|
||||
|
||||
def _get_block_mask_from_layout(self, layout: _InContextLayout) -> BlockMask:
|
||||
"""Cached flex_attention ``BlockMask`` over the padded [gen | reference-video] layout."""
|
||||
key = (
|
||||
layout.max_generation_video_num_tokens,
|
||||
layout.max_reference_video_num_tokens,
|
||||
layout.max_tokens_per_frame,
|
||||
str(self._device),
|
||||
)
|
||||
if key in self._block_masks:
|
||||
self._block_masks.move_to_end(key)
|
||||
return self._block_masks[key]
|
||||
if len(self._block_masks) >= 2:
|
||||
self._block_masks.popitem(last=False)
|
||||
|
||||
# _compile=True keeps the block-sparse layout consistent with the compiled
|
||||
# flex_attention; built once per key, so the compile cost is paid once.
|
||||
block_mask = create_block_mask(
|
||||
layout.block_mask_fn(),
|
||||
B=None,
|
||||
H=None,
|
||||
Q_LEN=layout.padded_generation_video_len,
|
||||
KV_LEN=layout.padded_kv_len,
|
||||
device=self._device,
|
||||
_compile=True,
|
||||
)
|
||||
self._block_masks[key] = block_mask
|
||||
return block_mask
|
||||
|
||||
def _patch_embeddings(
|
||||
self,
|
||||
latent_36ch: torch.Tensor, # [B, 36, f, H, W]
|
||||
) -> tuple[torch.Tensor, tuple[int, int, int]]:
|
||||
"""Patch-embed the latent into the token stream ``[B, f*h*w, C]`` and return the
|
||||
token grid_size ``(f, h, w)``; tokens are flattened row-major, w fastest."""
|
||||
p_t, p_h, p_w = self.patch_size
|
||||
_, _, num_frames, height, width = latent_36ch.shape
|
||||
grid_size = (num_frames // p_t, height // p_h, width // p_w)
|
||||
hidden_states = self.patch_embedding(latent_36ch)
|
||||
hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous()
|
||||
return hidden_states, grid_size
|
||||
|
||||
def _time_embeddings(
|
||||
self, timestep: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
timestep_embeddings = self.condition_embedder.time_embedder(timestep, None)
|
||||
timestep_modulation = self.condition_embedder.time_modulation(
|
||||
timestep_embeddings
|
||||
).unflatten(1, (6, -1))
|
||||
return timestep_embeddings, timestep_modulation
|
||||
|
||||
def prepare_context(
|
||||
self,
|
||||
prompt_embeddings: torch.Tensor,
|
||||
image_embeddings: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Project one branch's timestep-invariant context; owned by the current clip."""
|
||||
prompt_embeddings = self._pad_or_truncate_prompt_embeddings(prompt_embeddings)
|
||||
context = self.condition_embedder.text_embedder(prompt_embeddings)
|
||||
if image_embeddings is not None:
|
||||
image_context = self.condition_embedder.image_embedder(image_embeddings)
|
||||
context = torch.cat([image_context, context], dim=1)
|
||||
return context.to(self._dtype)
|
||||
|
||||
def _pad_or_truncate_prompt_embeddings(
|
||||
self, prompt_embeddings: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad or truncate the prompt embeddings (``[L, C]`` or ``[B, L, C]``) to
|
||||
``[B, max_text_len, C]``, as the official model does; cross-attention attends the pad rows too."""
|
||||
if prompt_embeddings.dim() == 2:
|
||||
prompt_embeddings = prompt_embeddings.unsqueeze(0) # [L, C] -> [1, L, C]
|
||||
|
||||
batch_size, num_tokens, channels = prompt_embeddings.shape
|
||||
if num_tokens < self.max_text_len:
|
||||
pad = prompt_embeddings.new_zeros(
|
||||
batch_size, self.max_text_len - num_tokens, channels
|
||||
)
|
||||
prompt_embeddings = torch.cat([prompt_embeddings, pad], dim=1)
|
||||
elif num_tokens > self.max_text_len:
|
||||
prompt_embeddings = prompt_embeddings[:, : self.max_text_len]
|
||||
return prompt_embeddings.to(self._dtype)
|
||||
|
||||
@torch.no_grad()
|
||||
def build_reference_kv(
|
||||
self, *, clip_cond: WanAnimate2ClipConditioning
|
||||
) -> WanAnimate2ReferenceKV:
|
||||
"""One forward_ref pass over the clip's reference video, returning every block's rotated K and V.
|
||||
|
||||
Text- and timestep-independent, so one result serves every step and CFG branch of the
|
||||
clip; the caller owns it and the DiT keeps no copy. Attention goes through USPAttention,
|
||||
so callers must wrap this in ``set_forward_context(...)`` like ``forward``.
|
||||
"""
|
||||
|
||||
# 36-channel reference-video input: fp32 latents (16) over the bf16 condition (20),
|
||||
# both cast to the param dtype so the cat runs in bf16 (see _make_generation_input).
|
||||
reference_video_latent_36ch = torch.cat(
|
||||
[
|
||||
clip_cond.reference_video_latents.to(self._dtype),
|
||||
clip_cond.reference_video_condition.to(self._dtype),
|
||||
],
|
||||
dim=0,
|
||||
).unsqueeze(0) # [1, 36, cf, H', W']
|
||||
reference_video_frame_0_image_embeddings = (
|
||||
clip_cond.reference_video_frame_0_image_embeddings
|
||||
) # [1, 257, 1280] bf16
|
||||
|
||||
# reference-video timestep is forced to 1.
|
||||
reference_video_timestep = torch.ones(1, device=self._device, dtype=torch.long)
|
||||
hidden_states, (reference_video_f, reference_video_h, reference_video_w) = (
|
||||
self._patch_embeddings(reference_video_latent_36ch)
|
||||
)
|
||||
_, timestep_modulation = self._time_embeddings(reference_video_timestep)
|
||||
reference_video_encoder_hidden_states = self.prepare_context(
|
||||
clip_cond.prompt_ref_embeddings, reference_video_frame_0_image_embeddings
|
||||
)
|
||||
|
||||
reference_video_rope_cos, reference_video_rope_sin = self._reference_video_rope(
|
||||
reference_video_f,
|
||||
reference_video_h,
|
||||
reference_video_w,
|
||||
generation_video_w=clip_cond.generation_video_grid_sizes[2],
|
||||
)
|
||||
reference_video_num_tokens = (
|
||||
reference_video_f * reference_video_h * reference_video_w
|
||||
)
|
||||
|
||||
# Ulysses SP: shard the reference-video sequence; each block a2a-gathers it and caches
|
||||
# full-seq / head-shard K/V. The reference-video hidden states are discarded, so no gather.
|
||||
# The gathered sequence is tail-padded to a multiple of the SP size; one key mask over
|
||||
# that padding serves every block (None when nothing is padded).
|
||||
reference_video_key_mask = None
|
||||
sp_size = get_sp_world_size()
|
||||
if sp_size > 1:
|
||||
hidden_states, _ = shard_seq(hidden_states, dim=1)
|
||||
gathered_seq_len = hidden_states.shape[1] * sp_size
|
||||
if gathered_seq_len != reference_video_num_tokens:
|
||||
reference_video_key_mask = _reference_video_key_mask(
|
||||
batch_size=hidden_states.shape[0],
|
||||
seq_len=gathered_seq_len,
|
||||
num_valid=reference_video_num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
k_by_block: dict[int, torch.Tensor] = {}
|
||||
v_by_block: dict[int, torch.Tensor] = {}
|
||||
# Call via __call__ (not forward_ref) so the layerwise-offload hooks fire.
|
||||
for idx, block in enumerate(self.blocks):
|
||||
hidden_states = block(
|
||||
kv_cache_mode="ref",
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=reference_video_encoder_hidden_states,
|
||||
timestep_modulation=timestep_modulation,
|
||||
reference_video_rope_cos=reference_video_rope_cos,
|
||||
reference_video_rope_sin=reference_video_rope_sin,
|
||||
reference_video_num_tokens=reference_video_num_tokens,
|
||||
index=idx,
|
||||
k_cache=k_by_block,
|
||||
v_cache=v_by_block,
|
||||
reference_video_key_mask=reference_video_key_mask,
|
||||
)
|
||||
return WanAnimate2ReferenceKV(k=k_by_block, v=v_by_block)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor, # [16, f, h, w] gen latent (B=1)
|
||||
encoder_hidden_states: torch.Tensor, # this CFG branch's prompt embeddings [L, C]
|
||||
timestep: torch.Tensor, # [1]
|
||||
# [1, 257, 1280] bf16 CLIP embed of the reference image
|
||||
encoder_hidden_states_image: torch.Tensor | None = None,
|
||||
guidance: torch.Tensor | None = None,
|
||||
*,
|
||||
# This clip's K/V from build_reference_kv; the same object for every step and CFG branch.
|
||||
reference_kv: WanAnimate2ReferenceKV,
|
||||
# This clip's conditioning from build_clip_conditioning.
|
||||
clip_cond: WanAnimate2ClipConditioning,
|
||||
projected_context: torch.Tensor | None = None,
|
||||
# The unconditional CFG branch skips block 9 (see _run_generation_blocks).
|
||||
is_unconditional: bool = False,
|
||||
) -> torch.Tensor:
|
||||
sp_size = get_sp_world_size()
|
||||
|
||||
latent_36ch = self._make_generation_input(
|
||||
hidden_states, clip_cond.generation_condition
|
||||
)
|
||||
hidden_states, (f, h, w) = self._patch_embeddings(latent_36ch)
|
||||
timestep_embeddings, timestep_modulation = self._time_embeddings(timestep)
|
||||
generation_encoder_hidden_states = (
|
||||
projected_context
|
||||
if projected_context is not None
|
||||
else self.prepare_context(
|
||||
encoder_hidden_states, encoder_hidden_states_image
|
||||
)
|
||||
)
|
||||
layout = _InContextLayout(
|
||||
(f, h, w),
|
||||
clip_cond.reference_video_grid_sizes,
|
||||
clip_cond.full_clip_grid_sizes,
|
||||
)
|
||||
|
||||
# No seq padding here: hidden_states is exactly f*h*w tokens and the in-context
|
||||
# layout pads internally. Under Ulysses SP each block a2a-gathers the full
|
||||
# sequence, so shard tail padding lands at the global tail where forward_gen
|
||||
# expects it.
|
||||
if sp_size > 1:
|
||||
hidden_states, _ = shard_seq(hidden_states, dim=1)
|
||||
hidden_states = self._run_generation_blocks(
|
||||
hidden_states,
|
||||
encoder_hidden_states=generation_encoder_hidden_states,
|
||||
timestep_modulation=timestep_modulation,
|
||||
layout=layout,
|
||||
reference_kv=reference_kv,
|
||||
is_unconditional=is_unconditional,
|
||||
)
|
||||
if sp_size > 1:
|
||||
# Gather the shards and trim tail padding back to the clip's token count.
|
||||
hidden_states = gather_seq(
|
||||
hidden_states, layout.generation_video_num_tokens, dim=1
|
||||
)
|
||||
|
||||
return self._unpatchify(hidden_states, timestep_embeddings, (f, h, w))
|
||||
|
||||
def _make_generation_input(
|
||||
self,
|
||||
latent: torch.Tensor, # [16, f, h, w] fp32
|
||||
generation_condition: torch.Tensor, # [20, f, h, w] bf16
|
||||
) -> torch.Tensor:
|
||||
"""36-channel DiT input ``[1, 36, f, h, w]``: the fp32 noisy latent over the bf16 generation condition."""
|
||||
# Both cast to the param dtype so the cat runs in bf16 and does not rely on autocast;
|
||||
# the fp32 latent would be rounded to bf16 by the patch embedding anyway.
|
||||
return torch.cat(
|
||||
[latent.to(self._dtype), generation_condition.to(self._dtype)], dim=0
|
||||
).unsqueeze(0)
|
||||
|
||||
def _run_generation_blocks(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
*,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
timestep_modulation: torch.Tensor,
|
||||
layout: _InContextLayout,
|
||||
reference_kv: WanAnimate2ReferenceKV,
|
||||
is_unconditional: bool,
|
||||
) -> torch.Tensor:
|
||||
"""forward_gen through every block with the in-context RoPE tables and block mask."""
|
||||
generation_video_rope_cos, generation_video_rope_sin = (
|
||||
self._generation_video_rope(
|
||||
layout.generation_video_f,
|
||||
layout.generation_video_h,
|
||||
layout.generation_video_w,
|
||||
)
|
||||
)
|
||||
block_mask = self._get_block_mask_from_layout(layout)
|
||||
|
||||
# Call via __call__ (not forward_gen) so the layerwise-offload hooks fire.
|
||||
for idx, block in enumerate(self.blocks):
|
||||
# Block 9 is dropped on the unconditional branch, as in the reference
|
||||
# wan_animate_2_model.forward_gen; the authors give no rationale.
|
||||
if is_unconditional and idx == 9:
|
||||
continue
|
||||
hidden_states = block(
|
||||
kv_cache_mode="gen",
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
timestep_modulation=timestep_modulation,
|
||||
gen_cos=generation_video_rope_cos,
|
||||
gen_sin=generation_video_rope_sin,
|
||||
block_mask=block_mask,
|
||||
layout=layout,
|
||||
index=idx,
|
||||
k_cache=reference_kv.k,
|
||||
v_cache=reference_kv.v,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
EntryClass = WanAnimate2Transformer3DModel
|
||||
@@ -0,0 +1,532 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Wan-Animate-2 DiT block: in-context self-attention over [gen tokens, then cached
|
||||
# reference-video K/V], its RoPE / Ulysses helpers, and the per-clip _InContextLayout
|
||||
# geometry.
|
||||
# Names kept from the reference implementation: the forward_ref / forward_gen methods and the
|
||||
# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image.
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Literal
|
||||
|
||||
import torch
|
||||
from torch.nn.attention.flex_attention import BlockMask, flex_attention
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_sp_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.usp import (
|
||||
_usp_input_all_to_all_qkv as _sp_gather_qkv,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.usp import (
|
||||
_usp_output_all_to_all,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformerBlock
|
||||
|
||||
# Eager flex_attention materializes the full [Q, KV] scores over the padded [gen | reference-video]
|
||||
# sequence (tens of GiB at video resolution); compiled, it is one fused kernel.
|
||||
_flex_attention_compiled = torch.compile(
|
||||
flex_attention,
|
||||
dynamic=False,
|
||||
mode="max-autotune",
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
)
|
||||
|
||||
|
||||
def _apply_rope_interleaved(
|
||||
x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Interleaved (GPT-J style) RoPE, identical to ``_apply_rotary_emb(is_neox_style=False)``;
|
||||
``x`` is ``[B, S, H, D]``, ``cos`` / ``sin`` are ``[S, D//2]`` fp32."""
|
||||
cos = cos[None, :, None, :] # [1, S, 1, D//2]
|
||||
sin = sin[None, :, None, :]
|
||||
x1 = x[..., 0::2].float()
|
||||
x2 = x[..., 1::2].float()
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
out = torch.stack([o1, o2], dim=-1).flatten(-2)
|
||||
return out.type_as(x)
|
||||
|
||||
|
||||
def _score_mod_impl(
|
||||
score: torch.Tensor,
|
||||
b_idx: torch.Tensor,
|
||||
h_idx: torch.Tensor,
|
||||
q_idx: torch.Tensor,
|
||||
kv_idx: torch.Tensor,
|
||||
hw: int,
|
||||
log_scale: float,
|
||||
) -> torch.Tensor:
|
||||
# Biases the keys of generation frame 1, [hw, 2*hw) in the padded layout, exactly as the
|
||||
# official Wan-Animate-2 code does; non-zero only when the checkpoint config sets log_scale.
|
||||
condition = (kv_idx >= hw) & (kv_idx < 2 * hw)
|
||||
return torch.where(condition, score + log_scale, score)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _make_score_mod(hw: int, log_scale: float) -> Callable[..., torch.Tensor]:
|
||||
# Stable object identity per (hw, log_scale): compiled flex_attention guards on the
|
||||
# score_mod object, so a fresh closure per call would recompile every clip.
|
||||
return partial(_score_mod_impl, hw=hw, log_scale=log_scale)
|
||||
|
||||
|
||||
def _sp_scatter_out(attn_output: torch.Tensor) -> torch.Tensor:
|
||||
"""Ulysses output a2a: [B, S_full, N_local, C] -> [B, S_local, N_tp, C]."""
|
||||
return _usp_output_all_to_all(attn_output, head_dim=2)
|
||||
|
||||
|
||||
def _reference_video_key_mask(
|
||||
*, batch_size: int, seq_len: int, num_valid: int, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""``[B, S]`` bool key mask: True for the leading ``num_valid`` reference-video tokens."""
|
||||
key_mask = torch.zeros((batch_size, seq_len), dtype=torch.bool, device=device)
|
||||
key_mask[:, :num_valid] = True
|
||||
return key_mask
|
||||
|
||||
|
||||
class WanAnimate2TransformerBlock(WanTransformerBlock):
|
||||
"""Wan block with the Wan-Animate-2 in-context self-attention.
|
||||
|
||||
No new parameters: forward_ref / forward_gen reuse the inherited modules and only
|
||||
route the self-attention differently.
|
||||
"""
|
||||
|
||||
def __init__(self, *args: Any, log_scale: float = 0.0, **kwargs: Any) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
if self.use_offline_qk_rotation:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 blocks do not apply the offline Q/K rotation that "
|
||||
"SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION selects for this checkpoint."
|
||||
)
|
||||
# Additive score bias on the keys of generation frame 1 (_score_mod_impl); 0.0 skips it.
|
||||
self.log_scale = log_scale
|
||||
|
||||
def _modulate(
|
||||
self, timestep_modulation: torch.Tensor
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""14B modulation split of ``timestep_modulation`` ``[B, 6, C]`` into six ``[B, 1, C]`` fp32 chunks; mirrors wanvideo.py."""
|
||||
assert timestep_modulation.dim() == 3, (
|
||||
"WanAnimate2TransformerBlock expects the 14B timestep modulation [B, 6, inner]; the ti2v "
|
||||
"seq-len modulation path is not used by Wan-Animate-2."
|
||||
)
|
||||
e = self.scale_shift_table + timestep_modulation.float()
|
||||
shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(
|
||||
6, dim=1
|
||||
)
|
||||
return shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa
|
||||
|
||||
def forward(
|
||||
self, *, kv_cache_mode: Literal["ref", "gen"], **kwargs: Any
|
||||
) -> torch.Tensor:
|
||||
"""Dispatch to forward_ref / forward_gen through ``nn.Module.__call__`` so the
|
||||
layerwise-offload module hooks fire for both passes; keyword arguments only."""
|
||||
if kv_cache_mode == "ref":
|
||||
return self.forward_ref(**kwargs)
|
||||
if kv_cache_mode == "gen":
|
||||
return self.forward_gen(**kwargs)
|
||||
raise ValueError(
|
||||
f"WanAnimate2TransformerBlock: unknown kv_cache_mode {kv_cache_mode!r} "
|
||||
"(expected 'ref' or 'gen')."
|
||||
)
|
||||
|
||||
def forward_ref(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor, # [B, S_reference_video, C] reference-video latent stream
|
||||
encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] reference-video context (CLIP tokens, then text)
|
||||
timestep_modulation: torch.Tensor, # [B, 6, C], reference-video timestep forced to 1
|
||||
reference_video_rope_cos: torch.Tensor, # reference-video RoPE cos [reference_video_num_tokens, D//2]
|
||||
reference_video_rope_sin: torch.Tensor,
|
||||
reference_video_num_tokens: int, # reference_video_f * reference_video_h * reference_video_w (valid reference-video tokens)
|
||||
index: int,
|
||||
k_cache: dict[int, torch.Tensor],
|
||||
v_cache: dict[int, torch.Tensor],
|
||||
# [B, S_reference_video] key mask for the SP tail padding, built once per clip by the
|
||||
# caller; None when the gathered sequence has no padding.
|
||||
reference_video_key_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Cache this block's rotated reference-video K and V, then run the reference-video self-attention
|
||||
(reference-video RoPE, pad keys masked) and return the updated reference-video stream."""
|
||||
orig_dtype = hidden_states.dtype
|
||||
shift, scale, gate, c_shift, c_scale, c_gate = self._modulate(
|
||||
timestep_modulation
|
||||
)
|
||||
|
||||
norm_hidden_states = self.norm1(hidden_states, shift, scale)
|
||||
query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
|
||||
|
||||
# Ulysses SP: gather the full reference-video sequence / shard heads before caching, so the
|
||||
# cache matches forward_gen's post-a2a layout.
|
||||
sp_size = get_sp_world_size()
|
||||
if sp_size > 1:
|
||||
query, key, value = _sp_gather_qkv(query, key, value)
|
||||
|
||||
# Reference-video self-attention. Under SP, S_reference_video is the gathered
|
||||
# (tail-padded) length: RoPE the valid slice and mask the pad keys. Without padding the
|
||||
# whole sequence is valid and no mask is passed: USPAttention serves a dense key mask
|
||||
# through SDPA instead of the configured backend.
|
||||
reference_video_seq_len = query.shape[1]
|
||||
if reference_video_seq_len == reference_video_num_tokens:
|
||||
q_roped = _apply_rope_interleaved(
|
||||
query, reference_video_rope_cos, reference_video_rope_sin
|
||||
)
|
||||
k_roped = _apply_rope_interleaved(
|
||||
key, reference_video_rope_cos, reference_video_rope_sin
|
||||
)
|
||||
key_mask = None
|
||||
else:
|
||||
q_roped = query.clone()
|
||||
k_roped = key.clone()
|
||||
q_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved(
|
||||
query[:, :reference_video_num_tokens],
|
||||
reference_video_rope_cos,
|
||||
reference_video_rope_sin,
|
||||
)
|
||||
k_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved(
|
||||
key[:, :reference_video_num_tokens],
|
||||
reference_video_rope_cos,
|
||||
reference_video_rope_sin,
|
||||
)
|
||||
key_mask = reference_video_key_mask
|
||||
if key_mask is None:
|
||||
key_mask = _reference_video_key_mask(
|
||||
batch_size=hidden_states.shape[0],
|
||||
seq_len=reference_video_seq_len,
|
||||
num_valid=reference_video_num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Reference positions are invariant within a clip. Keep the rotated key in the
|
||||
# original cache storage; SP's value can share the gathered QKV allocation.
|
||||
k_cache[index] = key.copy_(k_roped)
|
||||
v_cache[index] = value
|
||||
if sp_size > 1:
|
||||
# q/k/v are already gathered: skip USPAttention's own a2a, then scatter back.
|
||||
attn_output = self.attn1(
|
||||
q_roped,
|
||||
k_roped,
|
||||
value,
|
||||
attn_mask=key_mask,
|
||||
skip_sequence_parallel_override=True,
|
||||
)
|
||||
attn_output = _sp_scatter_out(attn_output)
|
||||
else:
|
||||
attn_output = self.attn1(q_roped, k_roped, value, attn_mask=key_mask)
|
||||
attn_output = attn_output.flatten(2)
|
||||
|
||||
return self._cross_and_ffn(
|
||||
hidden_states,
|
||||
attn_output,
|
||||
encoder_hidden_states,
|
||||
gate,
|
||||
c_shift,
|
||||
c_scale,
|
||||
c_gate,
|
||||
orig_dtype,
|
||||
)
|
||||
|
||||
def forward_gen(
|
||||
self,
|
||||
*,
|
||||
hidden_states: torch.Tensor, # [B, seq_len, C] gen stream (padded)
|
||||
encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] gen context (CLIP tokens, then text)
|
||||
timestep_modulation: torch.Tensor, # [B, 6, C]
|
||||
gen_cos: torch.Tensor, # gen-RoPE cos [generation_video_num_tokens, D//2]
|
||||
gen_sin: torch.Tensor,
|
||||
block_mask: BlockMask, # per full-length clip grid
|
||||
layout: _InContextLayout,
|
||||
index: int,
|
||||
k_cache: dict[int, torch.Tensor],
|
||||
v_cache: dict[int, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
"""One generation block with in-context masked self-attention over
|
||||
[gen tokens, then cached reference-video]."""
|
||||
orig_dtype = hidden_states.dtype
|
||||
shift, scale, gate, c_shift, c_scale, c_gate = self._modulate(
|
||||
timestep_modulation
|
||||
)
|
||||
|
||||
norm_hidden_states = self.norm1(hidden_states, shift, scale)
|
||||
query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
|
||||
|
||||
# Ulysses SP: gather the full gen sequence / shard heads. The cached reference-video K/V are
|
||||
# already in this layout.
|
||||
sp_size = get_sp_world_size()
|
||||
if sp_size > 1:
|
||||
query, key, value = _sp_gather_qkv(query, key, value)
|
||||
|
||||
# gen-RoPE on the valid gen tokens only (padding kept unrotated).
|
||||
num_generation_tokens = layout.generation_video_num_tokens
|
||||
q_roped = _apply_rope_interleaved(
|
||||
query[:, :num_generation_tokens], gen_cos, gen_sin
|
||||
)
|
||||
k_roped = _apply_rope_interleaved(
|
||||
key[:, :num_generation_tokens], gen_cos, gen_sin
|
||||
)
|
||||
if query.shape[1] != num_generation_tokens:
|
||||
q_roped = torch.cat([q_roped, query[:, num_generation_tokens:]], dim=1)
|
||||
k_roped = torch.cat([k_roped, key[:, num_generation_tokens:]], dim=1)
|
||||
|
||||
# Reference-video K is already rotated by forward_ref, including SP tail padding.
|
||||
reference_video_k = k_cache[index]
|
||||
reference_video_v = v_cache[index]
|
||||
|
||||
attn_output = self._in_context_attention(
|
||||
q_roped,
|
||||
k_roped,
|
||||
value,
|
||||
reference_video_k,
|
||||
reference_video_v,
|
||||
block_mask,
|
||||
layout,
|
||||
)
|
||||
if sp_size > 1:
|
||||
# scatter the sequence back / gather heads.
|
||||
attn_output = _sp_scatter_out(attn_output)
|
||||
attn_output = attn_output.flatten(2)
|
||||
|
||||
return self._cross_and_ffn(
|
||||
hidden_states,
|
||||
attn_output,
|
||||
encoder_hidden_states,
|
||||
gate,
|
||||
c_shift,
|
||||
c_scale,
|
||||
c_gate,
|
||||
orig_dtype,
|
||||
)
|
||||
|
||||
def _in_context_attention(
|
||||
self,
|
||||
q: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd)
|
||||
k: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd)
|
||||
v: torch.Tensor, # [B, seq_len, H, D]
|
||||
reference_video_k: torch.Tensor, # [B, reference_video_seq_len, H, D] (reference-video RoPE'd)
|
||||
reference_video_v: torch.Tensor, # [B, reference_video_seq_len, H, D]
|
||||
block_mask: BlockMask,
|
||||
layout: _InContextLayout,
|
||||
) -> torch.Tensor:
|
||||
"""Build the padded [gen | reference-video] layout and run compiled flex_attention.
|
||||
|
||||
The block-mask / score_mod are over sequence positions only, so TP composes
|
||||
trivially, and under Ulysses SP the callers hand in the full sequence with sharded
|
||||
heads, so the layout and mask are identical to sp=1."""
|
||||
batch_size, _, num_heads, head_dim = q.shape
|
||||
device, dtype = q.device, q.dtype
|
||||
padded_generation_video_len = layout.padded_generation_video_len
|
||||
padded_kv_len = layout.padded_kv_len
|
||||
|
||||
q_padding = q[:, layout.generation_video_num_tokens :]
|
||||
|
||||
q_padded = torch.zeros(
|
||||
batch_size,
|
||||
padded_generation_video_len,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
k_padded = torch.zeros(
|
||||
batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype
|
||||
)
|
||||
v_padded = torch.zeros(
|
||||
batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype
|
||||
)
|
||||
|
||||
num_latent_frames = layout.generation_video_f
|
||||
latent_hw = layout.generation_video_tokens_per_frame
|
||||
q_valid = q[:, : layout.generation_video_num_tokens].view(
|
||||
batch_size, num_latent_frames, latent_hw, num_heads, head_dim
|
||||
)
|
||||
k_valid = k[:, : layout.generation_video_num_tokens].view(
|
||||
batch_size, num_latent_frames, latent_hw, num_heads, head_dim
|
||||
)
|
||||
v_valid = v[:, : layout.generation_video_num_tokens].view(
|
||||
batch_size, num_latent_frames, latent_hw, num_heads, head_dim
|
||||
)
|
||||
max_tokens_per_frame = layout.max_tokens_per_frame
|
||||
q_padded[:, : num_latent_frames * max_tokens_per_frame].view(
|
||||
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
|
||||
)[:, :, :latent_hw] = q_valid
|
||||
k_padded[:, : num_latent_frames * max_tokens_per_frame].view(
|
||||
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
|
||||
)[:, :, :latent_hw] = k_valid
|
||||
v_padded[:, : num_latent_frames * max_tokens_per_frame].view(
|
||||
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
|
||||
)[:, :, :latent_hw] = v_valid
|
||||
|
||||
reference_video_f, reference_video_tokens_per_frame = (
|
||||
layout.reference_video_f,
|
||||
layout.reference_video_tokens_per_frame,
|
||||
)
|
||||
reference_video_k_valid = reference_video_k[
|
||||
:, : layout.reference_video_num_tokens
|
||||
].view(
|
||||
batch_size,
|
||||
reference_video_f,
|
||||
reference_video_tokens_per_frame,
|
||||
num_heads,
|
||||
head_dim,
|
||||
)
|
||||
reference_video_v_valid = reference_video_v[
|
||||
:, : layout.reference_video_num_tokens
|
||||
].view(
|
||||
batch_size,
|
||||
reference_video_f,
|
||||
reference_video_tokens_per_frame,
|
||||
num_heads,
|
||||
head_dim,
|
||||
)
|
||||
k_padded[
|
||||
:,
|
||||
padded_generation_video_len : padded_generation_video_len
|
||||
+ reference_video_f * max_tokens_per_frame,
|
||||
].view(
|
||||
batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim
|
||||
)[:, :, :reference_video_tokens_per_frame] = reference_video_k_valid
|
||||
v_padded[
|
||||
:,
|
||||
padded_generation_video_len : padded_generation_video_len
|
||||
+ reference_video_f * max_tokens_per_frame,
|
||||
].view(
|
||||
batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim
|
||||
)[:, :, :reference_video_tokens_per_frame] = reference_video_v_valid
|
||||
|
||||
# At 0.0 the bias is an exact identity, so the kernel is not asked to apply it;
|
||||
# otherwise the cached partial keeps the identity compiled flex guards on (_make_score_mod).
|
||||
log_scale = float(self.log_scale)
|
||||
_score_mod = (
|
||||
None
|
||||
if log_scale == 0.0
|
||||
else _make_score_mod(max_tokens_per_frame, log_scale)
|
||||
)
|
||||
|
||||
# flex_attention takes [B, H, L, D]; the default 1/sqrt(D) scale matches USPAttention.
|
||||
attn_output_padded = _flex_attention_compiled(
|
||||
q_padded.transpose(1, 2),
|
||||
k_padded.transpose(1, 2),
|
||||
v_padded.transpose(1, 2),
|
||||
block_mask=block_mask,
|
||||
score_mod=_score_mod,
|
||||
).transpose(1, 2) # -> [B, padded_generation_video_len, N, C]
|
||||
|
||||
attn_output_valid = attn_output_padded[
|
||||
:, : num_latent_frames * max_tokens_per_frame
|
||||
].view(
|
||||
batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
|
||||
)[:, :, :latent_hw]
|
||||
attn_output_valid = attn_output_valid.reshape(
|
||||
batch_size, num_latent_frames * latent_hw, num_heads, head_dim
|
||||
)
|
||||
if q_padding.shape[1] == 0:
|
||||
return attn_output_valid
|
||||
return torch.cat([attn_output_valid, q_padding], dim=1) # [B, seq_len, N, C]
|
||||
|
||||
|
||||
class _InContextLayout:
|
||||
"""Token geometry of the padded [generation | reference-video] in-context layout.
|
||||
|
||||
Three sets of sizes: this clip's token counts; those of a full-length clip, which every
|
||||
clip is padded up to; and the full-length sizes rounded up to 128 for flex_attention.
|
||||
The attention pads to the last set and the block mask is built from the same numbers,
|
||||
so one compiled mask serves every clip of a request."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
generation_video_grid: tuple[int, int, int],
|
||||
reference_video_grid: tuple[int, int, int],
|
||||
full_clip_grid: tuple[int, int, int],
|
||||
) -> None:
|
||||
f, h, w = generation_video_grid
|
||||
reference_video_f, reference_video_h, reference_video_w = reference_video_grid
|
||||
full_clip_f, full_clip_h, full_clip_w = full_clip_grid
|
||||
self.generation_video_f, self.generation_video_h, self.generation_video_w = (
|
||||
f,
|
||||
h,
|
||||
w,
|
||||
)
|
||||
self.reference_video_f, self.reference_video_h, self.reference_video_w = (
|
||||
reference_video_f,
|
||||
reference_video_h,
|
||||
reference_video_w,
|
||||
)
|
||||
|
||||
# This clip; shorter than full length only for the last clip of a video.
|
||||
self.generation_video_tokens_per_frame = h * w
|
||||
self.reference_video_tokens_per_frame = reference_video_h * reference_video_w
|
||||
self.generation_video_num_tokens = f * h * w
|
||||
self.reference_video_num_tokens = (
|
||||
reference_video_f * reference_video_h * reference_video_w
|
||||
)
|
||||
|
||||
# A full-length clip. The generation video carries one extra reference-image frame,
|
||||
# the reference video does not.
|
||||
self.max_tokens_per_frame = full_clip_h * full_clip_w
|
||||
self.max_generation_video_num_tokens = full_clip_f * self.max_tokens_per_frame
|
||||
self.max_reference_video_num_tokens = (
|
||||
full_clip_f - 1
|
||||
) * self.max_tokens_per_frame
|
||||
|
||||
# Padded to flex_attention's 128-token blocks; keys are [generation | reference video].
|
||||
self.padded_generation_video_len = (
|
||||
math.ceil(self.max_generation_video_num_tokens / 128) * 128
|
||||
)
|
||||
self.padded_reference_video_len = (
|
||||
math.ceil(self.max_reference_video_num_tokens / 128) * 128
|
||||
)
|
||||
self.padded_kv_len = (
|
||||
self.padded_generation_video_len + self.padded_reference_video_len
|
||||
)
|
||||
|
||||
def block_mask_fn(self) -> Callable[..., torch.Tensor]:
|
||||
"""Returns the block mask function based on the [gen | reference-video] in-context layout.
|
||||
|
||||
A valid gen query attends every valid gen key plus the reference-video keys of its own frame.
|
||||
|
||||
keys: [generation tokens | generation pad | reference tokens | reference pad]
|
||||
0 padded_generation_video_len padded_kv_len
|
||||
queries: [generation tokens | generation pad]
|
||||
0 padded_generation_video_len
|
||||
"""
|
||||
tokens_per_frame = self.max_tokens_per_frame
|
||||
num_generation_tokens = self.max_generation_video_num_tokens
|
||||
num_reference_tokens = self.max_reference_video_num_tokens
|
||||
reference_start = self.padded_generation_video_len
|
||||
|
||||
def _mask_function(
|
||||
b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
# Pad positions are not tokens; pad queries attend nothing.
|
||||
q_is_generation_token = q_idx < num_generation_tokens
|
||||
# Generation frame of the query; frame 0 is the reference image.
|
||||
q_frame = q_idx // tokens_per_frame
|
||||
|
||||
# Every generation token attends every generation token.
|
||||
kv_is_generation_token = kv_idx < num_generation_tokens
|
||||
|
||||
# Reference-video tokens are attended only by the queries of their own frame.
|
||||
kv_reference_idx = kv_idx - reference_start
|
||||
kv_is_reference_token = (kv_idx >= reference_start) & (
|
||||
kv_reference_idx < num_reference_tokens
|
||||
)
|
||||
# +1: the reference video has no counterpart for the generation stream's
|
||||
# reference-image frame 0.
|
||||
kv_reference_frame = kv_reference_idx // tokens_per_frame + 1
|
||||
kv_is_same_frame_reference_token = kv_is_reference_token & (
|
||||
q_frame == kv_reference_frame
|
||||
)
|
||||
return q_is_generation_token & (
|
||||
kv_is_generation_token | kv_is_same_frame_reference_token
|
||||
)
|
||||
|
||||
return _mask_function
|
||||
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Per-clip contracts between the Wan-Animate-2 stages and the DiT.
|
||||
|
||||
Lives on the model side so the DiT can type its ``clip_cond`` / ``reference_kv`` kwargs
|
||||
without importing a pipeline stage: the before-denoising stage builds the conditioning, the
|
||||
DiT's ``build_reference_kv`` turns it into the reference K/V the denoising stage hands to
|
||||
every ``forward`` of that clip.
|
||||
"""
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
|
||||
class WanAnimate2ClipConditioning(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Per-clip conditioning consumed by the DiT's forward_ref / forward_gen."""
|
||||
|
||||
# Inputs for ``forward_ref`` (reference video = the motion source)
|
||||
# VAE latents of this clip's reference-video frames,
|
||||
# [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32.
|
||||
reference_video_latents: torch.Tensor
|
||||
# Generation token grid (latent_t, latent_h // 2, latent_w // 2): the DiT patchifies the
|
||||
# latent with patch_size (1, 2, 2), so each spatial side halves.
|
||||
generation_video_grid_sizes: tuple[int, int, int]
|
||||
# CLIP tokens of the clip's first reference-video frame, [1, 257, 1280] bf16.
|
||||
reference_video_frame_0_image_embeddings: torch.Tensor
|
||||
# [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16.
|
||||
# Channels: [0:4] given-frame mask, all ones (every reference frame is known);
|
||||
# [4:20] VAE latent, the same tensor as reference_video_latents.
|
||||
# Time: the clip's reference-video latent frames; no reference-image frame in front.
|
||||
# forward_ref cats it under reference_video_latents (16 ch) for the 36-channel DiT input.
|
||||
reference_video_condition: torch.Tensor
|
||||
# T5 embedding of prompt_ref, [L_prompt_ref, 4096] bf16.
|
||||
prompt_ref_embeddings: torch.Tensor
|
||||
|
||||
# Inputs for: ``forward_gen``
|
||||
# [20, latent_t, latent_h, latent_w] bf16.
|
||||
# Channels: [0:4] given-frame mask (1 = known pixels, 0 = to generate);
|
||||
# [4:20] VAE latent of the given pixels, encoded zeros elsewhere.
|
||||
# Time: frame 0 is the reference image (mask = all ones); frames 1..latent_t are the clip's
|
||||
# own clip_latent_t frames, given only for the num_frames_conditioning frames fed
|
||||
# back from the previous clip (none for clip 0).
|
||||
# forward_gen cats it under the noise latent (16 ch) for the 36-channel DiT input.
|
||||
generation_condition: torch.Tensor
|
||||
# Reference-video token grid (reference_video_latent_t, _h // 2, _w // 2); same halving.
|
||||
reference_video_grid_sizes: tuple[int, int, int]
|
||||
# Token grid of a full-length clip (the configured clip_len); the attention layout and
|
||||
# block mask are sized from it even on the shorter last clip, so one compiled mask
|
||||
# serves the whole request.
|
||||
full_clip_grid_sizes: tuple[int, int, int]
|
||||
|
||||
# Pixel-frame count of this clip (the last clip may be shorter than clip_len).
|
||||
num_frames: int
|
||||
# Seeded initial noise for the denoising loop, [16, latent_t, latent_h, latent_w] fp32.
|
||||
init_noise: torch.Tensor
|
||||
|
||||
|
||||
class WanAnimate2ReferenceKV(msgspec.Struct, frozen=True):
|
||||
"""One clip's post-RoPE reference-video K and V, keyed by DiT block index.
|
||||
|
||||
Returned by ``WanAnimate2Transformer3DModel.build_reference_kv`` and owned by the caller
|
||||
for that clip only; the DiT keeps no copy between calls.
|
||||
"""
|
||||
|
||||
# [1, S_reference_video, local_heads, head_dim] in the DiT param dtype. Under Ulysses SP
|
||||
# this is the full sequence with sharded heads, the layout forward_gen uses after its a2a.
|
||||
k: dict[int, torch.Tensor]
|
||||
v: dict[int, torch.Tensor]
|
||||
@@ -617,6 +617,61 @@ class WanTransformerBlock(nn.Module):
|
||||
|
||||
self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
|
||||
|
||||
def _qkv(
|
||||
self, norm_hidden_states: torch.Tensor, local_heads: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
query, _ = self.to_q(norm_hidden_states)
|
||||
key, _ = self.to_k(norm_hidden_states)
|
||||
value, _ = self.to_v(norm_hidden_states)
|
||||
if self.norm_q is not None:
|
||||
query = (
|
||||
tensor_parallel_rms_norm(query, self.norm_q)
|
||||
if self.tp_rmsnorm
|
||||
else self.norm_q(query)
|
||||
)
|
||||
if self.norm_k is not None:
|
||||
key = (
|
||||
tensor_parallel_rms_norm(key, self.norm_k)
|
||||
if self.tp_rmsnorm
|
||||
else self.norm_k(key)
|
||||
)
|
||||
return tuple(
|
||||
tensor.squeeze(1).unflatten(2, (local_heads, self.dim_head))
|
||||
for tensor in (query, key, value)
|
||||
)
|
||||
|
||||
def _cross_and_ffn(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
attn_output: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
gate_msa: torch.Tensor,
|
||||
c_shift_msa: torch.Tensor,
|
||||
c_scale_msa: torch.Tensor,
|
||||
c_gate_msa: torch.Tensor,
|
||||
orig_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
attn_output, _ = self.to_out(attn_output)
|
||||
attn_output = attn_output.squeeze(1)
|
||||
null_shift = null_scale = torch.zeros(
|
||||
(1,), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
)
|
||||
norm_hidden_states, hidden_states = self.self_attn_residual_norm(
|
||||
hidden_states, attn_output, gate_msa, null_shift, null_scale
|
||||
)
|
||||
norm_hidden_states = norm_hidden_states.to(orig_dtype)
|
||||
hidden_states = hidden_states.to(orig_dtype)
|
||||
attn_output = self.attn2(
|
||||
norm_hidden_states, context=encoder_hidden_states, context_lens=None
|
||||
)
|
||||
norm_hidden_states, hidden_states = self.cross_attn_residual_norm(
|
||||
hidden_states, attn_output, 1, c_shift_msa, c_scale_msa
|
||||
)
|
||||
norm_hidden_states = norm_hidden_states.to(orig_dtype)
|
||||
hidden_states = hidden_states.to(orig_dtype)
|
||||
ff_output = self.ffn(norm_hidden_states)
|
||||
return self.mlp_residual(ff_output, c_gate_msa, hidden_states).to(orig_dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -655,23 +710,7 @@ class WanTransformerBlock(nn.Module):
|
||||
|
||||
# 1. Self-attention
|
||||
norm_hidden_states = self.norm1(hidden_states, shift_msa, scale_msa)
|
||||
query, _ = self.to_q(norm_hidden_states)
|
||||
key, _ = self.to_k(norm_hidden_states)
|
||||
value, _ = self.to_v(norm_hidden_states)
|
||||
|
||||
if self.norm_q is not None:
|
||||
if self.tp_rmsnorm:
|
||||
query = tensor_parallel_rms_norm(query, self.norm_q)
|
||||
else:
|
||||
query = self.norm_q(query)
|
||||
if self.norm_k is not None:
|
||||
if self.tp_rmsnorm:
|
||||
key = tensor_parallel_rms_norm(key, self.norm_k)
|
||||
else:
|
||||
key = self.norm_k(key)
|
||||
query = query.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
|
||||
key = key.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
|
||||
value = value.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
|
||||
query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
|
||||
|
||||
# Apply rotary embeddings
|
||||
cos, sin = freqs_cis
|
||||
@@ -729,38 +768,16 @@ class WanTransformerBlock(nn.Module):
|
||||
|
||||
attn_output = self.attn1(query, key, value)
|
||||
attn_output = attn_output.flatten(2)
|
||||
attn_output, _ = self.to_out(attn_output)
|
||||
attn_output = attn_output.squeeze(1)
|
||||
|
||||
null_shift = null_scale = torch.zeros(
|
||||
(1,), device=hidden_states.device, dtype=hidden_states.dtype
|
||||
return self._cross_and_ffn(
|
||||
hidden_states,
|
||||
attn_output,
|
||||
encoder_hidden_states,
|
||||
gate_msa,
|
||||
c_shift_msa,
|
||||
c_scale_msa,
|
||||
c_gate_msa,
|
||||
orig_dtype,
|
||||
)
|
||||
norm_hidden_states, hidden_states = self.self_attn_residual_norm(
|
||||
hidden_states, attn_output, gate_msa, null_shift, null_scale
|
||||
)
|
||||
norm_hidden_states, hidden_states = (
|
||||
norm_hidden_states.to(orig_dtype),
|
||||
hidden_states.to(orig_dtype),
|
||||
)
|
||||
|
||||
# 2. Cross-attention
|
||||
attn_output = self.attn2(
|
||||
norm_hidden_states, context=encoder_hidden_states, context_lens=None
|
||||
)
|
||||
norm_hidden_states, hidden_states = self.cross_attn_residual_norm(
|
||||
hidden_states, attn_output, 1, c_shift_msa, c_scale_msa
|
||||
)
|
||||
norm_hidden_states, hidden_states = (
|
||||
norm_hidden_states.to(orig_dtype),
|
||||
hidden_states.to(orig_dtype),
|
||||
)
|
||||
|
||||
# 3. Feed-forward
|
||||
ff_output = self.ffn(norm_hidden_states)
|
||||
hidden_states = self.mlp_residual(ff_output, c_gate_msa, hidden_states)
|
||||
hidden_states = hidden_states.to(orig_dtype)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class WanTransformerBlock_VSA(nn.Module):
|
||||
@@ -1057,29 +1074,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
)
|
||||
|
||||
# 3. Transformer blocks
|
||||
attn_backend = get_global_server_args().attention_backend
|
||||
transformer_block = (
|
||||
WanTransformerBlock_VSA
|
||||
if (attn_backend and attn_backend.lower() == "video_sparse_attn")
|
||||
else WanTransformerBlock
|
||||
)
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
transformer_block(
|
||||
inner_dim,
|
||||
config.ffn_dim,
|
||||
config.num_attention_heads,
|
||||
config.qk_norm,
|
||||
config.cross_attn_norm,
|
||||
config.eps,
|
||||
config.added_kv_proj_dim,
|
||||
self._supported_attention_backends
|
||||
| {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
|
||||
prefix=f"blocks.{i}",
|
||||
attention_type=config.attention_type,
|
||||
sla_topk=config.sla_topk,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self._make_block(index=i, config=config, quant_config=quant_config)
|
||||
for i in range(config.num_layers)
|
||||
]
|
||||
)
|
||||
@@ -1127,6 +1124,36 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
|
||||
self.layer_names = ["blocks"]
|
||||
|
||||
def _make_block(
|
||||
self,
|
||||
*,
|
||||
index: int,
|
||||
config: WanVideoConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> nn.Module:
|
||||
"""Block factory; subclasses override it to substitute their block class."""
|
||||
attn_backend = get_global_server_args().attention_backend
|
||||
transformer_block = (
|
||||
WanTransformerBlock_VSA
|
||||
if (attn_backend and attn_backend.lower() == "video_sparse_attn")
|
||||
else WanTransformerBlock
|
||||
)
|
||||
return transformer_block(
|
||||
config.num_attention_heads * config.attention_head_dim,
|
||||
config.ffn_dim,
|
||||
config.num_attention_heads,
|
||||
config.qk_norm,
|
||||
config.cross_attn_norm,
|
||||
config.eps,
|
||||
config.added_kv_proj_dim,
|
||||
self._supported_attention_backends
|
||||
| {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
|
||||
prefix=f"blocks.{index}",
|
||||
attention_type=config.attention_type,
|
||||
sla_topk=config.sla_topk,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _compute_rope_for_sequence_shard(
|
||||
self,
|
||||
@@ -1338,30 +1365,37 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
|
||||
if seq_shard_pad > 0:
|
||||
hidden_states = hidden_states[:, :seq_len_orig, :]
|
||||
|
||||
# 5. Output norm, projection & unpatchify
|
||||
if temb.dim() == 3:
|
||||
return self._unpatchify(
|
||||
hidden_states,
|
||||
temb,
|
||||
(post_patch_num_frames, post_patch_height, post_patch_width),
|
||||
)
|
||||
|
||||
def _unpatchify(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep_embeddings: torch.Tensor,
|
||||
grid_size: tuple[int, int, int],
|
||||
) -> torch.Tensor:
|
||||
"""Output norm, projection and token-grid reconstruction for Wan variants."""
|
||||
if timestep_embeddings.dim() == 3:
|
||||
# batch_size, seq_len, inner_dim (wan 2.2 ti2v)
|
||||
shift, scale = (
|
||||
self.scale_shift_table.unsqueeze(0) + temb.unsqueeze(2)
|
||||
self.scale_shift_table.unsqueeze(0) + timestep_embeddings.unsqueeze(2)
|
||||
).chunk(2, dim=2)
|
||||
shift = shift.squeeze(2)
|
||||
scale = scale.squeeze(2)
|
||||
else:
|
||||
# batch_size, inner_dim
|
||||
shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk(2, dim=1)
|
||||
shift, scale = (
|
||||
self.scale_shift_table + timestep_embeddings.unsqueeze(1)
|
||||
).chunk(2, dim=1)
|
||||
|
||||
hidden_states = self.norm_out(hidden_states, shift, scale)
|
||||
hidden_states, _ = self.proj_out(hidden_states)
|
||||
|
||||
hidden_states = hidden_states.reshape(
|
||||
batch_size,
|
||||
post_patch_num_frames,
|
||||
post_patch_height,
|
||||
post_patch_width,
|
||||
p_t,
|
||||
p_h,
|
||||
p_w,
|
||||
-1,
|
||||
hidden_states.shape[0], *grid_size, *self.patch_size, -1
|
||||
)
|
||||
hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6)
|
||||
output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3)
|
||||
|
||||
+36
-2
@@ -9,6 +9,7 @@
|
||||
# This wrapper delegates all numerical work to diffusers' implementation
|
||||
# and only adapts the interface for sglang's denoising stage.
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from diffusers import (
|
||||
DPMSolverMultistepScheduler as DiffusersDPMSolverMultistepScheduler,
|
||||
@@ -101,10 +102,43 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
|
||||
def begin_index(self) -> int | None:
|
||||
return self._begin_index
|
||||
|
||||
def set_timesteps(self, num_inference_steps: int, device=None, **kwargs):
|
||||
self._inner.set_timesteps(num_inference_steps, device=device, **kwargs)
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: int | None = None,
|
||||
device=None,
|
||||
sigmas: np.ndarray | list[float] | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
if sigmas is None:
|
||||
self._inner.set_timesteps(num_inference_steps, device=device, **kwargs)
|
||||
else:
|
||||
self._set_sigmas(np.asarray(sigmas, dtype=np.float64), device)
|
||||
self.timesteps = self._inner.timesteps
|
||||
|
||||
def _set_sigmas(self, sigmas: np.ndarray, device=None) -> None:
|
||||
inner = self._inner
|
||||
if inner.config.final_sigmas_type == "sigma_min":
|
||||
sigma_last = (
|
||||
(1 - inner.alphas_cumprod[0]) / inner.alphas_cumprod[0]
|
||||
) ** 0.5
|
||||
else:
|
||||
sigma_last = 0
|
||||
timesteps = sigmas * inner.config.num_train_timesteps
|
||||
sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32)
|
||||
inner.sigmas = torch.from_numpy(sigmas)
|
||||
inner.timesteps = torch.from_numpy(timesteps).to(
|
||||
device=device, dtype=torch.int64
|
||||
)
|
||||
inner.num_inference_steps = len(timesteps)
|
||||
self._reset_solver_state()
|
||||
|
||||
def _reset_solver_state(self) -> None:
|
||||
inner = self._inner
|
||||
inner.model_outputs = [None] * inner.config.solver_order
|
||||
inner.lower_order_nums = 0
|
||||
inner._step_index = None
|
||||
inner._begin_index = None
|
||||
|
||||
def scale_model_input(
|
||||
self, sample: torch.Tensor, timestep: int | None = None
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Wan-Animate-2 pipeline.
|
||||
|
||||
Stages: WanAnimate2BeforeDenoisingStage (per-clip conditioning) -> WanAnimate2DenoisingStage
|
||||
(per-clip denoise + in-loop VAE decode) -> WanAnimate2OutputStage (emit frames).
|
||||
Routed by model_index.json ``_class_name == "WanAnimate2Pipeline"``; config comes from
|
||||
the registry detector (Wan_Animate_2_14B_Config). Every component loads through the
|
||||
standard component loaders. Text uses TextEncodingStage; CLIP and VAE adapters
|
||||
preserve the reference preprocessing and latent scaling.
|
||||
"""
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
|
||||
TransformerLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2 import (
|
||||
WanAnimate2BeforeDenoisingStage,
|
||||
WanAnimate2DenoisingStage,
|
||||
WanAnimate2ImageEncoderAdapter,
|
||||
WanAnimate2OutputStage,
|
||||
WanAnimate2VaeAdapter,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class WanAnimate2TransformerLoader(TransformerLoader):
|
||||
"""The checkpoint is the plain Wan2.2-I2V-14B backbone, so every key must map."""
|
||||
|
||||
component_names = []
|
||||
strict_checkpoint_keys = True
|
||||
|
||||
|
||||
class WanAnimate2Pipeline(ComposedPipelineBase):
|
||||
"""Wan-Animate-2 14B in-context reference image + reference video -> video pipeline."""
|
||||
|
||||
pipeline_name = "WanAnimate2Pipeline"
|
||||
is_video_pipeline = True
|
||||
|
||||
# All seven model_index.json components. The scheduler is built from the
|
||||
# checkpoint's scheduler/scheduler_config.json (DPM-Solver++, flow prediction,
|
||||
# flow sigmas) by the shared loader. Two of its settings are overridden at
|
||||
# runtime: the stages call set_timesteps(sigmas=...) with the official Wan
|
||||
# sigma grid at every clip (sigma_0 is exactly 1.0), shifted by
|
||||
# pipeline_config.flow_shift, so the JSON's flow_shift and timestep_spacing
|
||||
# are not consulted. The JSON's beta_schedule only shapes the beta-derived
|
||||
# tables, which the flow-sigma path does not read.
|
||||
_required_config_modules: list[str] = [
|
||||
"text_encoder",
|
||||
"tokenizer",
|
||||
"vae",
|
||||
"transformer",
|
||||
"image_encoder",
|
||||
"image_processor",
|
||||
"scheduler",
|
||||
]
|
||||
component_loaders = {"transformer": WanAnimate2TransformerLoader}
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
|
||||
# Keep native modules visible to residency and layerwise-offload management;
|
||||
# adapters only translate the model-specific stage API.
|
||||
device = get_local_torch_device()
|
||||
image_encoder = WanAnimate2ImageEncoderAdapter.from_loaded(
|
||||
self.get_module("image_encoder"),
|
||||
image_processor=self.get_module("image_processor"),
|
||||
device=device,
|
||||
)
|
||||
vae = WanAnimate2VaeAdapter(self.get_module("vae"))
|
||||
self.add_stage(
|
||||
stage_name="wan_animate_2_before_denoising",
|
||||
stage=WanAnimate2BeforeDenoisingStage(
|
||||
vae=vae,
|
||||
image_encoder=image_encoder,
|
||||
text_encoder=self.get_module("text_encoder"),
|
||||
tokenizer=self.get_module("tokenizer"),
|
||||
pipeline_config=server_args.pipeline_config,
|
||||
scheduler=self.get_module("scheduler"),
|
||||
),
|
||||
)
|
||||
# Decode each clip before constructing the following clip's conditions.
|
||||
self.add_stage(
|
||||
stage_name="wan_animate_2_denoising",
|
||||
stage=WanAnimate2DenoisingStage(
|
||||
transformer=self.get_module("transformer"),
|
||||
scheduler=self.get_module("scheduler"),
|
||||
pipeline=self,
|
||||
vae=vae,
|
||||
image_encoder=image_encoder,
|
||||
),
|
||||
)
|
||||
# Replaces the standard DecodingStage: frames are already decoded in the loop.
|
||||
self.add_stage_factory(
|
||||
RoleType.DECODER,
|
||||
lambda: WanAnimate2OutputStage(),
|
||||
"decoding_stage",
|
||||
)
|
||||
|
||||
|
||||
EntryClass = WanAnimate2Pipeline
|
||||
@@ -234,8 +234,31 @@ class DecodingStage(PipelineStage):
|
||||
|
||||
Returns:
|
||||
Decoded video tensor with shape (batch, channels, frames, height, width),
|
||||
normalized to [0, 1] range and moved to CPU as float32
|
||||
normalized to [0, 1] on the decode device.
|
||||
"""
|
||||
normalized_on_frames = (
|
||||
(lambda frames: on_frames(_to_unit_range(frames)))
|
||||
if on_frames is not None
|
||||
else None
|
||||
)
|
||||
image = self.decode_raw(
|
||||
latents,
|
||||
server_args,
|
||||
vae_dtype=vae_dtype,
|
||||
on_frames=normalized_on_frames,
|
||||
)
|
||||
return _to_unit_range(image)
|
||||
|
||||
@torch.no_grad()
|
||||
def decode_raw(
|
||||
self,
|
||||
latents: torch.Tensor,
|
||||
server_args: ServerArgs,
|
||||
*,
|
||||
vae_dtype: torch.dtype,
|
||||
on_frames=None,
|
||||
) -> torch.Tensor:
|
||||
"""Decode without pixel normalization, including for inter-clip conditioning."""
|
||||
latents = latents.to(get_local_torch_device())
|
||||
# The caller resolves the decode-only override before component use so
|
||||
# residency and execution agree on the target dtype.
|
||||
@@ -270,9 +293,7 @@ class DecodingStage(PipelineStage):
|
||||
latents = latents.to(vae_dtype)
|
||||
decode_kwargs = {}
|
||||
if on_frames is not None:
|
||||
decode_kwargs["on_frames"] = lambda frames: on_frames(
|
||||
_to_unit_range(frames)
|
||||
)
|
||||
decode_kwargs["on_frames"] = on_frames
|
||||
with temporary_module_dtype(
|
||||
self.vae, vae_dtype, enabled=should_cast_vae
|
||||
) as vae:
|
||||
@@ -304,8 +325,6 @@ class DecodingStage(PipelineStage):
|
||||
raise
|
||||
image = _ensure_tensor_decode_output(decode_output)
|
||||
|
||||
# De-normalize image to [0, 1] range
|
||||
image = _to_unit_range(image)
|
||||
return image
|
||||
|
||||
def load_model(self):
|
||||
|
||||
@@ -2084,11 +2084,40 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
neg_cond_kwargs=ctx.neg_cond_kwargs,
|
||||
guidance=ctx.guidance,
|
||||
)
|
||||
timesteps_cpu = self._run_denoising_loop(ctx, batch, server_args)
|
||||
|
||||
# collect rollout outputs before finalization gathers or replaces latents
|
||||
if batch.rollout:
|
||||
self._postprocess_rollout_outputs(
|
||||
batch=batch,
|
||||
latents=ctx.latents,
|
||||
num_inference_steps=len(timesteps_cpu),
|
||||
final_timestep=timesteps_cpu.new_zeros(()),
|
||||
server_args=server_args,
|
||||
)
|
||||
self._finalize_denoising_loop(ctx, batch, server_args)
|
||||
return batch
|
||||
|
||||
def _run_denoising_loop(
|
||||
self,
|
||||
ctx: DenoisingContext,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
*,
|
||||
collect_trajectory: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Run a prepared loop, including profiling and DiT residency.
|
||||
|
||||
Clip-based models can reuse this without finalizing the entire request.
|
||||
"""
|
||||
denoising_start_time = time.time()
|
||||
self._before_denoising_loop(ctx, batch, server_args)
|
||||
# to avoid device-sync caused by timestep comparison
|
||||
timesteps_cpu = ctx.timesteps.cpu()
|
||||
num_timesteps = timesteps_cpu.shape[0]
|
||||
progress_step_interval = ctx.extra.get(
|
||||
"progress_step_interval", ctx.scheduler.order
|
||||
)
|
||||
# Re-resolve the explicit-range gate so the per-step markers
|
||||
# below honor this request's is_warmup state. Layer hooks are
|
||||
# registered by the residency manager at the use-site.
|
||||
@@ -2132,7 +2161,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
# BEFORE _run_denoising_step so ctx.latents is still the
|
||||
# pre-step value. Gated on batch.rollout to keep the
|
||||
# non-rollout path strictly untouched.
|
||||
if batch.rollout:
|
||||
if collect_trajectory and batch.rollout:
|
||||
batch._rollout_loop_step_index = step_index
|
||||
self._maybe_append_dit_trajectory_step(
|
||||
batch=batch,
|
||||
@@ -2141,11 +2170,12 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
step_index=step_index,
|
||||
)
|
||||
self._run_denoising_step(ctx, step, batch, server_args)
|
||||
self._record_trajectory(ctx, step, batch, server_args)
|
||||
if collect_trajectory:
|
||||
self._record_trajectory(ctx, step, batch, server_args)
|
||||
|
||||
if step_index == num_timesteps - 1 or (
|
||||
(step_index + 1) > ctx.num_warmup_steps
|
||||
and (step_index + 1) % ctx.scheduler.order == 0
|
||||
and (step_index + 1) % progress_step_interval == 0
|
||||
and progress_bar is not None
|
||||
):
|
||||
progress_bar.update()
|
||||
@@ -2165,20 +2195,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
del step
|
||||
self._finish_active_component_use()
|
||||
|
||||
# Rollout postprocessing must run BEFORE _finalize_denoising_loop so
|
||||
# the final scheduler.step output (ctx.latents) is still SP-sharded and
|
||||
# can be gathered uniformly alongside the per-step dit_trajectory via
|
||||
# gather_stacked_latents_for_sp.
|
||||
if batch.rollout:
|
||||
self._postprocess_rollout_outputs(
|
||||
batch=batch,
|
||||
latents=ctx.latents,
|
||||
num_inference_steps=num_timesteps,
|
||||
final_timestep=timesteps_cpu.new_zeros(()),
|
||||
server_args=server_args,
|
||||
)
|
||||
self._finalize_denoising_loop(ctx, batch, server_args)
|
||||
return batch
|
||||
return timesteps_cpu
|
||||
|
||||
def _get_extra_func_kwarg_names(self, func) -> tuple[bool, frozenset[str]]:
|
||||
import functools
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Wan-Animate-2-specific pipeline stages."""
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
|
||||
WanAnimate2BeforeDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
|
||||
WanAnimate2DenoisingStage,
|
||||
WanAnimate2OutputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
|
||||
WanAnimate2ImageEncoderAdapter,
|
||||
WanAnimate2VaeAdapter,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"WanAnimate2BeforeDenoisingStage",
|
||||
"WanAnimate2DenoisingStage",
|
||||
"WanAnimate2OutputStage",
|
||||
"WanAnimate2ImageEncoderAdapter",
|
||||
"WanAnimate2VaeAdapter",
|
||||
]
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Reference-video audio extraction for the Wan-Animate-2 before-denoising stage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.multimodal.audio_from_video import decode_audio_container
|
||||
|
||||
# Logger callables with the ``PipelineStage.log_*`` signature: ``(msg, *args)``.
|
||||
LogFn = Callable[..., None]
|
||||
|
||||
|
||||
def extract_reference_video_audio(
|
||||
reference_video_path: str,
|
||||
*,
|
||||
log_info: LogFn,
|
||||
log_warning: LogFn,
|
||||
log_no_audio_track: LogFn | None = None,
|
||||
) -> tuple[torch.Tensor | None, int | None]:
|
||||
"""Return ``(audio, sample_rate)`` for ``OutputBatch.audio``: audio ``[1, C, L]`` fp32 in
|
||||
[-1, 1] (dim 0 is the per-output axis the save path indexes) at the track's native sample
|
||||
rate and channel count, or ``(None, None)`` when the video has no decodable audio track.
|
||||
The track is decoded in-process by the shared ``decode_audio_container`` reader (PyAV).
|
||||
Best-effort: every failure is logged, none is raised. A video without an audio track goes
|
||||
to ``log_no_audio_track`` (``log_warning`` when None); a track that cannot be decoded goes
|
||||
to ``log_warning`` with the reader's message."""
|
||||
try:
|
||||
sample_rate = _audio_track_sample_rate(reference_video_path)
|
||||
if sample_rate is None:
|
||||
(log_no_audio_track or log_warning)(
|
||||
"Wan-Animate-2: reference video %s has no audio track; the output is silent.",
|
||||
reference_video_path,
|
||||
)
|
||||
return None, None
|
||||
# [L, C] fp32 in [-1, 1]; the reader needs a target rate, so the native one keeps the
|
||||
# track unresampled.
|
||||
samples = decode_audio_container(
|
||||
reference_video_path, target_sr=sample_rate, mono=False
|
||||
)
|
||||
except Exception as e: # audio is best-effort: a failure never fails the request
|
||||
log_warning(
|
||||
"Wan-Animate-2: cannot decode the audio track of reference video %s (%s); "
|
||||
"the output is silent.",
|
||||
reference_video_path,
|
||||
e,
|
||||
)
|
||||
return None, None
|
||||
|
||||
audio = torch.from_numpy(samples).T.contiguous().unsqueeze(0) # [1, C, L]
|
||||
log_info(
|
||||
"Wan-Animate-2: extracted reference-video audio %s @ %d Hz.",
|
||||
tuple(audio.shape),
|
||||
sample_rate,
|
||||
)
|
||||
return audio, sample_rate
|
||||
|
||||
|
||||
def _audio_track_sample_rate(path: str) -> int | None:
|
||||
"""Native sample rate of the video's first audio stream; None when it has none."""
|
||||
import av
|
||||
|
||||
with av.open(path) as container:
|
||||
if not container.streams.audio:
|
||||
return None
|
||||
return int(container.streams.audio[0].rate)
|
||||
+706
@@ -0,0 +1,706 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Wan-Animate-2 before-denoising stage.
|
||||
|
||||
Builds the per-clip conditioning container (``WanAnimate2ClipConditioning``) the DiT
|
||||
reads via its ``clip_cond`` kwarg, plus the clip-invariant text/CLIP/VAE encodings.
|
||||
Preprocessing helpers live in ``preprocess.py`` next to this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import cv2
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import torch
|
||||
from einops import rearrange
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
get_sp_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
|
||||
WanAnimate2ClipConditioning,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
|
||||
DPMSolverMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio import (
|
||||
extract_reference_video_audio,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
|
||||
LetterboxInfo,
|
||||
get_padding_len,
|
||||
make_conditioning_mask,
|
||||
read_reference_video_frames,
|
||||
resize_by_area,
|
||||
validate_and_get_single_string,
|
||||
zigzag_padding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import (
|
||||
Wan_Animate_2_14B_Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
|
||||
WanAnimate2ImageEncoderAdapter,
|
||||
WanAnimate2VaeAdapter,
|
||||
)
|
||||
|
||||
# Frames of the previous clip's decoded output that open the next clip as given (not denoised);
|
||||
# the official pipelines/wan_animate_2_pipeline.py hard-codes num_frames_conditioning = 1.
|
||||
_NUM_FRAMES_CONDITIONING = 1
|
||||
# Wan VAE temporal compression: 4 pixel frames per latent frame after the first.
|
||||
_VAE_TEMPORAL_COMPRESSION = 4
|
||||
# Wan VAE latent channels.
|
||||
_NUM_LATENT_CHANNELS = 16
|
||||
|
||||
|
||||
def generation_token_grid(
|
||||
num_frames: int, latent_h: int, latent_w: int
|
||||
) -> tuple[int, int, int]:
|
||||
"""DiT token grid (f, h, w) of a clip of ``num_frames`` pixel frames. Wan VAE: 4k + 1 pixel
|
||||
frames -> k + 1 latent frames; one latent frame is prepended for the reference image;
|
||||
the DiT patchifies with patch_size (1, 2, 2), so each spatial side halves."""
|
||||
clip_latent_t = (num_frames - 1) // _VAE_TEMPORAL_COMPRESSION + 1
|
||||
return clip_latent_t + 1, latent_h // 2, latent_w // 2
|
||||
|
||||
|
||||
def _generation_latent_shape(
|
||||
latent_h: int, latent_w: int, num_frames: int
|
||||
) -> tuple[int, int, int, int]:
|
||||
"""``[16, latent_t, latent_h, latent_w]`` of the latent the DiT denoises for a clip of
|
||||
``num_frames`` pixel frames; latent_t comes from generation_token_grid (clip frames plus
|
||||
the leading reference-image slot)."""
|
||||
latent_t = generation_token_grid(num_frames, latent_h, latent_w)[0]
|
||||
return (_NUM_LATENT_CHANNELS, latent_t, latent_h, latent_w)
|
||||
|
||||
|
||||
def get_sampling_sigmas(sampling_steps: int, shift: float) -> np.ndarray:
|
||||
"""Official Wan sampler grid: ``[sampling_steps]`` fp64 sigmas, uniform on [1, 0) with ``shift`` applied (sigma_0 == 1)."""
|
||||
sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps]
|
||||
sigma = shift * sigma / (1 + (shift - 1) * sigma)
|
||||
return sigma
|
||||
|
||||
|
||||
class _PerClipDenoisingMetadata(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Schedule metadata for one clip of the reference video."""
|
||||
|
||||
# Position in the schedule, 0-based.
|
||||
clip_index: int
|
||||
# Start frame (inclusive) in the padded reference video.
|
||||
frame_start_index: int
|
||||
# End frame (exclusive) in the padded reference video.
|
||||
frame_end_index: int
|
||||
# Pixel-frame count, frame_end_index - frame_start_index, zigzag-padded frames included;
|
||||
# shorter than clip_len only on the last clip.
|
||||
num_frames: int
|
||||
# Leading frames taken from the previous clip's output: 0 for clip 0, else
|
||||
# num_frames_conditioning.
|
||||
num_frames_conditioning_for_clip: int
|
||||
|
||||
|
||||
def build_schedule(
|
||||
reference_video_frames: list[np.ndarray],
|
||||
clip_len: int,
|
||||
num_frames_conditioning: int,
|
||||
) -> list[_PerClipDenoisingMetadata]:
|
||||
"""Prepare denoising metadata for all the clips of the reference video."""
|
||||
schedule: list[_PerClipDenoisingMetadata] = []
|
||||
start = 0
|
||||
clip_index = 0
|
||||
num_frames_reference_video = len(reference_video_frames)
|
||||
while True:
|
||||
if start + num_frames_conditioning >= num_frames_reference_video:
|
||||
break
|
||||
num_frames_conditioning_for_clip = 0 if start == 0 else num_frames_conditioning
|
||||
clip_len_current = clip_len
|
||||
if num_frames_reference_video - start < clip_len: # last clip shrinks
|
||||
clip_len_current = num_frames_reference_video - start
|
||||
schedule.append(
|
||||
_PerClipDenoisingMetadata(
|
||||
clip_index=clip_index,
|
||||
frame_start_index=start,
|
||||
frame_end_index=start + clip_len_current,
|
||||
num_frames=clip_len_current,
|
||||
num_frames_conditioning_for_clip=num_frames_conditioning_for_clip,
|
||||
)
|
||||
)
|
||||
start += clip_len_current - num_frames_conditioning
|
||||
clip_index += 1
|
||||
if not schedule:
|
||||
raise ValueError(
|
||||
"Wan-Animate-2 reference video produced 0 clips; check "
|
||||
"clip_len/num_frames_conditioning and that the video decoded frames."
|
||||
)
|
||||
return schedule
|
||||
|
||||
|
||||
class _WanAnimate2Inputs(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""The per-request inputs read off the Req."""
|
||||
|
||||
reference_image_path: str
|
||||
reference_video_path: str
|
||||
|
||||
prompt: str
|
||||
prompt_ref: str
|
||||
negative_prompt: str
|
||||
|
||||
width: int
|
||||
height: int
|
||||
clip_len: int
|
||||
num_frames_conditioning: int
|
||||
fps: int
|
||||
seed: int
|
||||
num_inference_steps: int
|
||||
guidance_scale: float
|
||||
enable_audio: bool
|
||||
|
||||
@classmethod
|
||||
def from_request(cls, batch: Req) -> _WanAnimate2Inputs:
|
||||
"""Read the Wan-Animate-2 input fields off the Req (all SamplingParam-backed)."""
|
||||
reference_image_path = validate_and_get_single_string(
|
||||
batch.image_path, "image_path"
|
||||
)
|
||||
reference_video_path = validate_and_get_single_string(
|
||||
batch.video_path, "video_path"
|
||||
)
|
||||
prompt = validate_and_get_single_string(batch.prompt, "prompt")
|
||||
prompt_ref = validate_and_get_single_string(batch.prompt_ref, "prompt_ref")
|
||||
# negative_prompt_embeddings is always computed; an unset negative prompt encodes "".
|
||||
negative_prompt = validate_and_get_single_string(
|
||||
batch.negative_prompt or "", "negative_prompt"
|
||||
)
|
||||
|
||||
seed = batch.seed
|
||||
if isinstance(seed, list) and len(seed) != 1:
|
||||
raise ValueError(
|
||||
f"Wan-Animate-2 takes exactly one 'seed' per request, got {len(seed)}"
|
||||
)
|
||||
if isinstance(seed, list):
|
||||
seed = seed[0]
|
||||
|
||||
return _WanAnimate2Inputs(
|
||||
reference_image_path=reference_image_path,
|
||||
reference_video_path=reference_video_path,
|
||||
prompt=prompt,
|
||||
prompt_ref=prompt_ref,
|
||||
negative_prompt=negative_prompt,
|
||||
width=int(batch.width),
|
||||
height=int(batch.height),
|
||||
clip_len=int(batch.clip_len),
|
||||
num_frames_conditioning=_NUM_FRAMES_CONDITIONING,
|
||||
fps=int(batch.fps),
|
||||
seed=int(seed),
|
||||
num_inference_steps=int(batch.num_inference_steps),
|
||||
guidance_scale=float(batch.guidance_scale),
|
||||
enable_audio=bool(batch.enable_audio),
|
||||
)
|
||||
|
||||
|
||||
class WanAnimate2RequestState(msgspec.Struct, kw_only=True):
|
||||
"""Clip-independent state, computed once, reused by every clip."""
|
||||
|
||||
device: torch.device
|
||||
sp_size: int
|
||||
generator: torch.Generator
|
||||
inputs: _WanAnimate2Inputs
|
||||
|
||||
# reference image
|
||||
letterbox_info: LetterboxInfo
|
||||
reference_image_height: int
|
||||
reference_image_width: int
|
||||
latent_h: int # reference_image_height // 8
|
||||
latent_w: int # reference_image_width // 8
|
||||
reference_image_condition: torch.Tensor # [20, 1, latent_h, latent_w] bf16
|
||||
reference_image_embeddings: torch.Tensor # [1, 257, 1280] bf16
|
||||
|
||||
# text
|
||||
prompt_embeddings: torch.Tensor # [L, 4096] bf16
|
||||
negative_prompt_embeddings: torch.Tensor # [L, 4096] bf16
|
||||
prompt_ref_embeddings: torch.Tensor # [L_prompt_ref, 4096] bf16
|
||||
|
||||
# reference video, zigzag-padded to the clip schedule
|
||||
reference_video_frames: list[np.ndarray] # resized [H, W, 3] uint8 RGB frames
|
||||
schedule: list[_PerClipDenoisingMetadata]
|
||||
# frame count before padding; the final video is trimmed to it
|
||||
num_reference_video_frames: int
|
||||
|
||||
# Token grid of a full-length clip; every clip's attention layout is padded to it.
|
||||
full_clip_grid_sizes: tuple[int, int, int]
|
||||
|
||||
# Reference-video audio, opt-in per request via enable_audio; None gives a silent mp4.
|
||||
audio: torch.Tensor | None = None # [1, C, L] fp32 in [-1, 1]
|
||||
audio_sample_rate: int | None = None
|
||||
# Assembled by the denoising stage for the output stage, [T, H, W, C] uint8.
|
||||
decoded_frames: np.ndarray | None = None
|
||||
# Initial noise for clip 0, drawn by the before-denoising stage so that batch.latents is
|
||||
# the real denoising input; later clips draw from the generator in schedule order.
|
||||
clip_0_init_noise: torch.Tensor | None = None
|
||||
|
||||
|
||||
# The one batch.extra entry Wan-Animate-2 uses; later stages read it back through
|
||||
# request_state_from_batch.
|
||||
WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY = "wan_animate_2_request_state"
|
||||
|
||||
|
||||
def request_state_from_batch(batch: Req) -> WanAnimate2RequestState:
|
||||
state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
|
||||
if not isinstance(state, WanAnimate2RequestState):
|
||||
raise ValueError(
|
||||
f"batch.extra[{WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY!r}] is missing: "
|
||||
"WanAnimate2BeforeDenoisingStage must run before this stage."
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
def _mirror_request_state_into_batch(
|
||||
batch: Req,
|
||||
request_state: WanAnimate2RequestState,
|
||||
*,
|
||||
scheduler: DPMSolverMultistepScheduler,
|
||||
flow_shift: float,
|
||||
) -> None:
|
||||
"""Populate the standard DenoisingStage input fields from the request state.
|
||||
|
||||
The clip loop keeps reading ``request_state``; the embeddings, generator, sigma grid and
|
||||
timesteps exist for the shared stage verification, warmup and residency bookkeeping.
|
||||
``batch.latents`` is clip 0's initial noise, drawn here (the seed's first draw) and
|
||||
consumed by ``build_clip_conditioning`` for clip 0, so the sequence of generator draws is
|
||||
the same as when the loop drew it: numerics are unchanged."""
|
||||
inputs = request_state.inputs
|
||||
batch.prompt_embeds = [request_state.prompt_embeddings]
|
||||
batch.negative_prompt_embeds = [request_state.negative_prompt_embeddings]
|
||||
batch.image_embeds = [request_state.reference_image_embeddings]
|
||||
batch.generator = request_state.generator
|
||||
|
||||
sigmas = get_sampling_sigmas(inputs.num_inference_steps, flow_shift)
|
||||
scheduler.set_timesteps(sigmas=sigmas, device=request_state.device)
|
||||
batch.timesteps = scheduler.timesteps
|
||||
batch.sigmas = sigmas.tolist()
|
||||
|
||||
batch.latents = _draw_clip_0_init_noise(request_state)
|
||||
batch.raw_latent_shape = batch.latents.shape
|
||||
|
||||
|
||||
def _draw_clip_0_init_noise(request_state: WanAnimate2RequestState) -> torch.Tensor:
|
||||
"""Draw clip 0's initial noise from the request generator (its first draw) and keep it on
|
||||
the request state for build_clip_conditioning; clips 1.. then take the following draws."""
|
||||
init_noise = torch.randn(
|
||||
*_generation_latent_shape(
|
||||
request_state.latent_h,
|
||||
request_state.latent_w,
|
||||
request_state.schedule[0].num_frames,
|
||||
),
|
||||
dtype=torch.float32,
|
||||
device=request_state.device,
|
||||
generator=request_state.generator,
|
||||
)
|
||||
request_state.clip_0_init_noise = init_noise
|
||||
return init_noise
|
||||
|
||||
|
||||
VaeEncoderFn = Callable[[list[torch.Tensor]], list[torch.Tensor]]
|
||||
ImageEmbedderFn = Callable[[list[torch.Tensor]], torch.Tensor]
|
||||
|
||||
|
||||
def build_clip_conditioning(
|
||||
request_state: WanAnimate2RequestState,
|
||||
clip_denoising_metadata: _PerClipDenoisingMetadata,
|
||||
prev_clip_conditioning_frames: torch.Tensor | None,
|
||||
*,
|
||||
vae_encoder: VaeEncoderFn,
|
||||
image_embedder: ImageEmbedderFn,
|
||||
) -> WanAnimate2ClipConditioning:
|
||||
"""Build conditioning for a single clip."""
|
||||
device = request_state.device
|
||||
latent_h, latent_w = request_state.latent_h, request_state.latent_w
|
||||
reference_image_height, reference_image_width = (
|
||||
request_state.reference_image_height,
|
||||
request_state.reference_image_width,
|
||||
)
|
||||
|
||||
# gen/noise latent geometry
|
||||
num_frames = clip_denoising_metadata.num_frames
|
||||
grid_sizes = generation_token_grid(num_frames, latent_h, latent_w)
|
||||
latent_t = grid_sizes[0]
|
||||
# The clip's own latent frames, without the prepended reference-image frame.
|
||||
clip_latent_t = latent_t - 1
|
||||
init_noise_shape = _generation_latent_shape(latent_h, latent_w, num_frames)
|
||||
|
||||
with (
|
||||
torch.autocast(device_type=device.type, dtype=torch.bfloat16),
|
||||
torch.no_grad(),
|
||||
):
|
||||
# previous-clip condition over the clip's own latent frames (no reference-image frame)
|
||||
# [4, clip_latent_t, latent_h, latent_w] fp32
|
||||
prev_clip_condition_mask = make_conditioning_mask(
|
||||
clip_latent_t,
|
||||
latent_h,
|
||||
latent_w,
|
||||
clip_denoising_metadata.num_frames_conditioning_for_clip,
|
||||
device=device,
|
||||
)
|
||||
# The clip's pixel video: the given frames first, zeros where the DiT generates.
|
||||
# Built on ``device`` so the frames never leave the GPU.
|
||||
prev_clip_condition_pixel_space = torch.zeros(
|
||||
3,
|
||||
num_frames,
|
||||
reference_image_height,
|
||||
reference_image_width,
|
||||
device=device,
|
||||
)
|
||||
if clip_denoising_metadata.num_frames_conditioning_for_clip > 0:
|
||||
# prev_clip_conditioning_frames: the previous clip's last
|
||||
# ``num_frames_conditioning`` decoded frames,
|
||||
# [C, num_frames_conditioning, reference_image_height, reference_image_width] bf16 in [-1, 1].
|
||||
if prev_clip_conditioning_frames is None:
|
||||
raise ValueError(
|
||||
"For all clips except clip-0 `prev_clip_conditioning_frames` must be not None."
|
||||
)
|
||||
|
||||
prev_clip_condition_pixel_space[
|
||||
:, : clip_denoising_metadata.num_frames_conditioning_for_clip
|
||||
] = torch.nn.functional.interpolate(
|
||||
prev_clip_conditioning_frames[
|
||||
:, : clip_denoising_metadata.num_frames_conditioning_for_clip
|
||||
],
|
||||
size=(reference_image_height, reference_image_width),
|
||||
mode="bicubic",
|
||||
)
|
||||
|
||||
# [16, clip_latent_t, latent_h, latent_w] fp32
|
||||
prev_clip_condition = vae_encoder([prev_clip_condition_pixel_space])[0]
|
||||
|
||||
# [20, clip_latent_t, latent_h, latent_w] bf16
|
||||
prev_clip_condition = torch.concat(
|
||||
[prev_clip_condition_mask, prev_clip_condition]
|
||||
).to(torch.bfloat16)
|
||||
|
||||
# [20, latent_t, latent_h, latent_w] bf16
|
||||
# recall: latent_t = clip_latent_t + 1 (+1 is for the reference image condition in latent space)
|
||||
generation_condition = torch.concat(
|
||||
[request_state.reference_image_condition, prev_clip_condition], dim=1
|
||||
)
|
||||
|
||||
# reference-video slice -> reference_video_latents
|
||||
# [t, h, w, c] uint8
|
||||
reference_video_clip_np = np.stack(
|
||||
request_state.reference_video_frames[
|
||||
clip_denoising_metadata.frame_start_index : clip_denoising_metadata.frame_end_index
|
||||
]
|
||||
)
|
||||
reference_video_pixels = rearrange(
|
||||
torch.from_numpy(reference_video_clip_np).to(device), "t h w c -> c t h w"
|
||||
)
|
||||
# [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, t, h, w]
|
||||
reference_video_pixels = (reference_video_pixels.float() / 127.5 - 1).to(
|
||||
torch.bfloat16
|
||||
)
|
||||
|
||||
# [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32
|
||||
# Stays fp32 (vae.encode returns .float()); forward_ref cats it with the
|
||||
# bf16 reference_video_condition under autocast type promotion.
|
||||
reference_video_latents = vae_encoder([reference_video_pixels])[0]
|
||||
(
|
||||
_,
|
||||
reference_video_latent_t,
|
||||
reference_video_latent_h,
|
||||
reference_video_latent_w,
|
||||
) = reference_video_latents.shape
|
||||
reference_video_grid_sizes = (
|
||||
reference_video_latent_t,
|
||||
reference_video_latent_h // 2,
|
||||
reference_video_latent_w // 2,
|
||||
)
|
||||
|
||||
# [3, reference_video_height, reference_video_width] bf16
|
||||
reference_video_frame_0 = reference_video_pixels[:, 0]
|
||||
reference_video_frame_0_image_embeddings = image_embedder(
|
||||
[reference_video_frame_0[:, None, :, :]]
|
||||
).to(torch.bfloat16) # [1, 257, 1280] bf16
|
||||
|
||||
# reference_video_condition (forward_ref): mask, then VAE latent of the reference-video slice.
|
||||
# Every frame of the slice is given, so the latent is reference_video_latents itself;
|
||||
# the official code encodes the same pixels a second time here.
|
||||
reference_video_condition_mask = make_conditioning_mask(
|
||||
reference_video_latent_t,
|
||||
reference_video_latent_h,
|
||||
reference_video_latent_w,
|
||||
num_frames,
|
||||
device=device,
|
||||
)
|
||||
# [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16
|
||||
reference_video_condition = torch.concat(
|
||||
[reference_video_condition_mask, reference_video_latents]
|
||||
).to(torch.bfloat16)
|
||||
|
||||
# Drawn from the request's seeded generator; clips are built in schedule order, so
|
||||
# clip k always gets the k-th draw and (seed, clip_index) identifies the noise. Clip 0's
|
||||
# draw already happened in the before-denoising stage (it is batch.latents).
|
||||
if (
|
||||
clip_denoising_metadata.clip_index == 0
|
||||
and request_state.clip_0_init_noise is not None
|
||||
):
|
||||
init_noise = request_state.clip_0_init_noise
|
||||
else:
|
||||
init_noise = torch.randn(
|
||||
*init_noise_shape,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
generator=request_state.generator,
|
||||
)
|
||||
|
||||
return WanAnimate2ClipConditioning(
|
||||
reference_video_latents=reference_video_latents,
|
||||
generation_video_grid_sizes=grid_sizes,
|
||||
reference_video_frame_0_image_embeddings=reference_video_frame_0_image_embeddings,
|
||||
reference_video_condition=reference_video_condition,
|
||||
prompt_ref_embeddings=request_state.prompt_ref_embeddings,
|
||||
generation_condition=generation_condition,
|
||||
reference_video_grid_sizes=reference_video_grid_sizes,
|
||||
full_clip_grid_sizes=request_state.full_clip_grid_sizes,
|
||||
num_frames=clip_denoising_metadata.num_frames,
|
||||
init_noise=init_noise,
|
||||
)
|
||||
|
||||
|
||||
class WanAnimate2BeforeDenoisingStage(TextEncodingStage):
|
||||
"""Native text encoding plus the model-specific reference and clip conditioning."""
|
||||
|
||||
# Text-only stage dedup must not skip fresh image/video conditions or the request RNG.
|
||||
deduplicated_output_fields = ()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vae: WanAnimate2VaeAdapter,
|
||||
image_encoder: WanAnimate2ImageEncoderAdapter,
|
||||
text_encoder: torch.nn.Module,
|
||||
tokenizer,
|
||||
pipeline_config: Wan_Animate_2_14B_Config,
|
||||
scheduler: DPMSolverMultistepScheduler,
|
||||
) -> None:
|
||||
super().__init__(text_encoders=[text_encoder], tokenizers=[tokenizer])
|
||||
self.vae = vae
|
||||
self.image_encoder = image_encoder
|
||||
self.pipeline_config = pipeline_config
|
||||
self._silent_reference_logged = False
|
||||
self._audio_failure_logged = False
|
||||
# Shared with the denoising stage, which resets it per clip; used here only to derive
|
||||
# batch.timesteps on the same sigma grid.
|
||||
self.scheduler = scheduler
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
name = self._component_stage_name(stage_name)
|
||||
return super().component_uses(server_args, stage_name) + [
|
||||
ComponentUse(stage_name=name, component_name="vae"),
|
||||
ComponentUse(stage_name=name, component_name="image_encoder"),
|
||||
]
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
device = get_local_torch_device()
|
||||
sp_size = max(1, int(get_sp_world_size()))
|
||||
|
||||
inputs = _WanAnimate2Inputs.from_request(batch)
|
||||
|
||||
# One seeded generator per request; clips draw their noise from it in order.
|
||||
generator = torch.Generator(device=device).manual_seed(inputs.seed)
|
||||
|
||||
audio, audio_sample_rate = self._extract_audio(inputs)
|
||||
request_state = self._build_request_state(
|
||||
inputs,
|
||||
device,
|
||||
sp_size,
|
||||
generator,
|
||||
server_args,
|
||||
audio=audio,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
)
|
||||
if len(request_state.schedule) > 1:
|
||||
self.log_info(
|
||||
"reference video splits into %d clips; each is conditioned in the denoising "
|
||||
"loop via build_clip_conditioning.",
|
||||
len(request_state.schedule),
|
||||
)
|
||||
batch.extra[WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY] = request_state
|
||||
_mirror_request_state_into_batch(
|
||||
batch,
|
||||
request_state,
|
||||
scheduler=self.scheduler,
|
||||
flow_shift=self.pipeline_config.flow_shift,
|
||||
)
|
||||
return batch
|
||||
|
||||
def _build_request_state(
|
||||
self,
|
||||
inputs: _WanAnimate2Inputs,
|
||||
device: torch.device,
|
||||
sp_size: int,
|
||||
generator: torch.Generator,
|
||||
server_args: ServerArgs,
|
||||
*,
|
||||
audio: torch.Tensor | None,
|
||||
audio_sample_rate: int | None,
|
||||
) -> WanAnimate2RequestState:
|
||||
width, height = inputs.width, inputs.height
|
||||
|
||||
reference_image = cv2.imread(inputs.reference_image_path)
|
||||
if reference_image is None:
|
||||
raise ValueError(
|
||||
f"Wan-Animate-2: failed to read reference image at {inputs.reference_image_path!r} "
|
||||
"(cv2.imread returned None: missing file or unsupported/corrupt image)."
|
||||
)
|
||||
reference_image = reference_image[..., ::-1] # BGR -> RGB
|
||||
reference_image, letterbox_info = resize_by_area(
|
||||
reference_image, width * height, divisor=16
|
||||
)
|
||||
|
||||
# reference video: decode, fps resample, zigzag pad.
|
||||
frames = read_reference_video_frames(inputs.reference_video_path, inputs.fps)
|
||||
reference_video_frames = [
|
||||
resize_by_area(f, width * height, divisor=16)[0] for f in frames
|
||||
]
|
||||
num_reference_video_frames = len(frames)
|
||||
target_len = get_padding_len(num_reference_video_frames, inputs.clip_len)
|
||||
reference_video_frames = zigzag_padding(reference_video_frames, target_len)
|
||||
|
||||
# Keep the official padding and geometry of every needed clip, but do not
|
||||
# denoise clips whose entire output would be discarded by the final trim.
|
||||
schedule = [
|
||||
clip
|
||||
for clip in build_schedule(
|
||||
reference_video_frames, inputs.clip_len, inputs.num_frames_conditioning
|
||||
)
|
||||
if clip.frame_start_index + clip.num_frames_conditioning_for_clip
|
||||
< num_reference_video_frames
|
||||
]
|
||||
|
||||
# reference-image encodings (constant across clips)
|
||||
reference_image_height, reference_image_width = (
|
||||
reference_image.shape[0],
|
||||
reference_image.shape[1],
|
||||
)
|
||||
latent_h, latent_w = reference_image_height // 8, reference_image_width // 8
|
||||
|
||||
reference_image_pixel_values = rearrange(
|
||||
torch.from_numpy(reference_image).to(device), "h w c -> c 1 h w"
|
||||
)
|
||||
# [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, 1, H, W]
|
||||
reference_image_pixel_values = (
|
||||
reference_image_pixel_values.float() / 127.5 - 1
|
||||
).to(torch.bfloat16)
|
||||
|
||||
with (
|
||||
torch.autocast(device_type=device.type, dtype=torch.bfloat16),
|
||||
torch.no_grad(),
|
||||
):
|
||||
# [16, 1, latent_h, latent_w] fp32
|
||||
reference_image_latents = self.vae_encoder([reference_image_pixel_values])[
|
||||
0
|
||||
]
|
||||
|
||||
# [4, 1, latent_h, latent_w] fp32
|
||||
reference_image_mask = make_conditioning_mask(
|
||||
1, latent_h, latent_w, 1, device=device
|
||||
)
|
||||
|
||||
# [20, 1, latent_h, latent_w] bf16
|
||||
reference_image_condition = torch.concat(
|
||||
[reference_image_mask, reference_image_latents]
|
||||
).to(torch.bfloat16)
|
||||
|
||||
reference_image_embeddings = self.image_embedder(
|
||||
[reference_image_pixel_values]
|
||||
).to(torch.bfloat16) # [1, 257, 1280] bf16
|
||||
|
||||
# Keep batch-one encoder arithmetic while sharing native caching and residency.
|
||||
prompt_embeddings, prompt_ref_embeddings, negative_prompt_embeddings = [
|
||||
self.encode_text(prompt, server_args, device=device)[0][0]
|
||||
for prompt in (inputs.prompt, inputs.prompt_ref, inputs.negative_prompt)
|
||||
]
|
||||
|
||||
return WanAnimate2RequestState(
|
||||
device=device,
|
||||
sp_size=sp_size,
|
||||
generator=generator,
|
||||
inputs=inputs,
|
||||
letterbox_info=letterbox_info,
|
||||
reference_image_height=reference_image_height,
|
||||
reference_image_width=reference_image_width,
|
||||
latent_h=latent_h,
|
||||
latent_w=latent_w,
|
||||
reference_image_condition=reference_image_condition,
|
||||
reference_image_embeddings=reference_image_embeddings,
|
||||
prompt_embeddings=prompt_embeddings,
|
||||
negative_prompt_embeddings=negative_prompt_embeddings,
|
||||
prompt_ref_embeddings=prompt_ref_embeddings,
|
||||
reference_video_frames=reference_video_frames,
|
||||
schedule=schedule,
|
||||
num_reference_video_frames=num_reference_video_frames,
|
||||
full_clip_grid_sizes=generation_token_grid(
|
||||
inputs.clip_len, latent_h, latent_w
|
||||
),
|
||||
audio=audio,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
)
|
||||
|
||||
def _extract_audio(
|
||||
self, inputs: _WanAnimate2Inputs
|
||||
) -> tuple[torch.Tensor | None, int | None]:
|
||||
"""Per-request opt-out; any failure yields (None, None), a silent video."""
|
||||
if not inputs.enable_audio:
|
||||
return None, None
|
||||
return extract_reference_video_audio(
|
||||
inputs.reference_video_path,
|
||||
log_info=self.log_info,
|
||||
log_no_audio_track=self._log_silent_reference,
|
||||
log_warning=self._log_audio_failure,
|
||||
)
|
||||
|
||||
def _log_silent_reference(self, msg: str, *args: Any) -> None:
|
||||
# A reference video without an audio track is normal input, not a failure:
|
||||
# one info line per process, then debug.
|
||||
if self._silent_reference_logged:
|
||||
self.log_debug(msg, *args)
|
||||
return
|
||||
self._silent_reference_logged = True
|
||||
self.log_info(
|
||||
msg + " Set enable_audio=false on the request to skip audio extraction.",
|
||||
*args,
|
||||
)
|
||||
|
||||
def _log_audio_failure(self, msg: str, *args: Any) -> None:
|
||||
# A track the audio reader cannot decode is the same for every request: warn once
|
||||
# per process with the remedy, then keep the detail at debug.
|
||||
if self._audio_failure_logged:
|
||||
self.log_debug(msg, *args)
|
||||
return
|
||||
self._audio_failure_logged = True
|
||||
self.log_warning(
|
||||
msg
|
||||
+ " Set enable_audio=false on the request to skip audio extraction; further "
|
||||
"audio failures are logged at debug level.",
|
||||
*args,
|
||||
)
|
||||
|
||||
def vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
|
||||
with self.use_declared_component(component_name="vae", module=self.vae.vae):
|
||||
return self.vae.encode(videos)
|
||||
|
||||
def image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor:
|
||||
with self.use_declared_component(
|
||||
component_name="image_encoder", module=self.image_encoder.model
|
||||
):
|
||||
return self.image_encoder.visual(videos)
|
||||
+532
@@ -0,0 +1,532 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Wan-Animate-2 denoising and output stages.
|
||||
|
||||
WanAnimate2DenoisingStage orchestrates clips around the shared denoising loop: the in-context DiT
|
||||
takes per-clip conditioning via its ``clip_cond`` kwarg and the clip's reference K/V via
|
||||
``reference_kv`` (built once per clip with ``build_reference_kv`` and dropped with the clip),
|
||||
runs cond/uncond CFG as two forwards (the unconditional branch skips block 9 via
|
||||
``is_unconditional``), resets the scheduler per clip on the official sigma grid, and
|
||||
VAE-decodes each clip in-loop to feed the next.
|
||||
WanAnimate2OutputStage then emits the assembled frames.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_local_torch_device,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import (
|
||||
run_cfg_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import (
|
||||
CFGBranch,
|
||||
CFGPolicy,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_classifier_free_guidance_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
|
||||
WanAnimate2ClipConditioning,
|
||||
WanAnimate2ReferenceKV,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
|
||||
DPMSolverMultistepScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
|
||||
DenoisingContext,
|
||||
DenoisingStage,
|
||||
DenoisingStepState,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
|
||||
WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY,
|
||||
WanAnimate2RequestState,
|
||||
build_clip_conditioning,
|
||||
get_sampling_sigmas,
|
||||
request_state_from_batch,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
|
||||
WanAnimate2ImageEncoderAdapter,
|
||||
WanAnimate2VaeAdapter,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
|
||||
LetterboxInfo,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
V,
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
|
||||
|
||||
def _is_request_state(value: object) -> bool:
|
||||
return isinstance(value, WanAnimate2RequestState)
|
||||
|
||||
|
||||
def _is_decoded_frames(value: object) -> bool:
|
||||
"""``[T, H, W, C]`` uint8 frames assembled by the denoising stage."""
|
||||
return isinstance(value, np.ndarray) and value.ndim == 4 and value.dtype == np.uint8
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class WanAnimate2DenoisingContext(DenoisingContext):
|
||||
clip_condition: WanAnimate2ClipConditioning
|
||||
reference_image_embeddings: torch.Tensor
|
||||
cfg_parallel: bool
|
||||
guidance_scale: float
|
||||
reference_kv: WanAnimate2ReferenceKV | None = None
|
||||
context_by_branch: dict[str, torch.Tensor] = field(default_factory=dict)
|
||||
|
||||
|
||||
class WanAnimate2DenoisingStage(DenoisingStage):
|
||||
"""Wan-Animate-2 in-context denoising over all reference-video clips.
|
||||
|
||||
Clip 0 uses the precomputed conditioning; clips > 0 are built from the previous clip's decoded
|
||||
frames, denoised, decoded in-loop, overlap-trimmed, and assembled into the full video.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
transformer: nn.Module,
|
||||
scheduler: DPMSolverMultistepScheduler,
|
||||
pipeline: ComposedPipelineBase,
|
||||
vae: WanAnimate2VaeAdapter,
|
||||
image_encoder: WanAnimate2ImageEncoderAdapter,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
transformer=transformer, scheduler=scheduler, pipeline=pipeline, vae=vae
|
||||
)
|
||||
self.vae: WanAnimate2VaeAdapter = vae
|
||||
self.clip_decoder = DecodingStage(vae, pipeline=pipeline)
|
||||
# Clips > 0 are conditioned inside this stage, so it holds the image encoder too.
|
||||
self.image_encoder = image_encoder
|
||||
|
||||
def _vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
|
||||
with self.use_declared_component(component_name="vae", module=self.vae.vae):
|
||||
return self.vae.encode(videos)
|
||||
|
||||
def _image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor:
|
||||
with self.use_declared_component(
|
||||
component_name="image_encoder", module=self.image_encoder.model
|
||||
):
|
||||
return self.image_encoder.visual(videos)
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.min_dims(1)])
|
||||
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(4)])
|
||||
result.add_check("raw_latent_shape", batch.raw_latent_shape, V.not_none)
|
||||
result.add_check("sigmas", batch.sigmas, V.list_not_empty)
|
||||
result.add_check(
|
||||
"prompt_embeds", batch.prompt_embeds, V.list_of_tensors_dims(2)
|
||||
)
|
||||
result.add_check(
|
||||
"negative_prompt_embeds",
|
||||
batch.negative_prompt_embeds,
|
||||
V.list_of_tensors_dims(2),
|
||||
)
|
||||
result.add_check("image_embeds", batch.image_embeds, V.list_of_tensors_dims(3))
|
||||
result.add_check(
|
||||
"num_inference_steps", batch.num_inference_steps, V.positive_int
|
||||
)
|
||||
result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float)
|
||||
result.add_check("generator", batch.generator, V.generator_or_list_generators)
|
||||
result.add_check(
|
||||
WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY,
|
||||
batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY),
|
||||
_is_request_state,
|
||||
)
|
||||
return result
|
||||
|
||||
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
|
||||
result.add_check(
|
||||
"decoded_frames",
|
||||
(
|
||||
request_state.decoded_frames
|
||||
if isinstance(request_state, WanAnimate2RequestState)
|
||||
else None
|
||||
),
|
||||
_is_decoded_frames,
|
||||
)
|
||||
return result
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
# build_clip_conditioning runs CLIP.visual inside this stage, so image_encoder must stay
|
||||
# resident (it is CPU-offloaded by default for video tasks).
|
||||
uses = super().component_uses(server_args, stage_name)
|
||||
# The VAE adapter runs the VAE in fp32 with autocast off; do not let the manager
|
||||
# cast it to the configured vae precision on the way in.
|
||||
uses = [
|
||||
replace(use, target_dtype=None) if use.component_name == "vae" else use
|
||||
for use in uses
|
||||
]
|
||||
if not any(use.component_name == "image_encoder" for use in uses):
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name=self._component_stage_name(stage_name),
|
||||
component_name="image_encoder",
|
||||
)
|
||||
)
|
||||
return uses
|
||||
|
||||
@torch.no_grad()
|
||||
def _denoise_single_clip(
|
||||
self,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
request_state: WanAnimate2RequestState,
|
||||
clip_condition: WanAnimate2ClipConditioning,
|
||||
) -> torch.Tensor:
|
||||
"""Denoise one clip; return the latents with the reference-image slot dropped.
|
||||
``batch`` is handed to set_forward_context, which the attention backends read, and
|
||||
supplies the metrics object the per-step records go to."""
|
||||
device = get_local_torch_device()
|
||||
|
||||
prompt_embeddings = request_state.prompt_embeddings
|
||||
negative_prompt_embeddings = request_state.negative_prompt_embeddings
|
||||
reference_image_embeddings = request_state.reference_image_embeddings
|
||||
|
||||
# Official sampler grid: sigma_0 is exactly 1.0, so sigmas are set explicitly.
|
||||
# set_timesteps also resets all solver state for the new clip.
|
||||
num_inference_steps = request_state.inputs.num_inference_steps
|
||||
sample_shift = server_args.pipeline_config.flow_shift
|
||||
self.scheduler.set_timesteps(
|
||||
sigmas=get_sampling_sigmas(num_inference_steps, sample_shift), device=device
|
||||
)
|
||||
|
||||
guidance_scale = request_state.inputs.guidance_scale
|
||||
|
||||
# negative_prompt_embeddings is always computed (an unset prompt encodes ""), so CFG
|
||||
# depends on the scale alone, unlike Req.validate which also needs a negative prompt.
|
||||
perform_cfg = guidance_scale > 1
|
||||
|
||||
# CFG parallel: cond on cfg_rank 0, uncond on cfg_rank 1, then all-gather. Only
|
||||
# with --enable-cfg-parallel and initialized model-parallel groups.
|
||||
cfg_parallel = (
|
||||
perform_cfg
|
||||
and bool(server_args.enable_cfg_parallel)
|
||||
and model_parallel_is_initialized()
|
||||
)
|
||||
if cfg_parallel and get_classifier_free_guidance_world_size() != 2:
|
||||
logger.warning_once(
|
||||
"CFG parallel enabled but the CFG group world size is "
|
||||
f"{get_classifier_free_guidance_world_size()} (Wan-Animate-2 expects 2 "
|
||||
"for its cond/uncond split); using the sequential two-pass CFG path."
|
||||
)
|
||||
cfg_parallel = False
|
||||
branches = [
|
||||
CFGBranch(
|
||||
"conditional",
|
||||
True,
|
||||
{
|
||||
"encoder_hidden_states": prompt_embeddings,
|
||||
"is_unconditional": False,
|
||||
},
|
||||
)
|
||||
]
|
||||
if perform_cfg:
|
||||
branches.append(
|
||||
CFGBranch(
|
||||
"unconditional",
|
||||
False,
|
||||
{
|
||||
"encoder_hidden_states": negative_prompt_embeddings,
|
||||
"is_unconditional": True,
|
||||
},
|
||||
)
|
||||
)
|
||||
cfg_policy = CFGPolicy(branches=branches)
|
||||
|
||||
ctx = WanAnimate2DenoisingContext(
|
||||
scheduler=self.scheduler,
|
||||
extra_step_kwargs={},
|
||||
target_dtype=torch.bfloat16,
|
||||
autocast_enabled=True,
|
||||
timesteps=self.scheduler.timesteps,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_warmup_steps=0,
|
||||
image_kwargs={},
|
||||
pos_cond_kwargs={},
|
||||
neg_cond_kwargs={},
|
||||
latents=clip_condition.init_noise,
|
||||
boundary_timestep=None,
|
||||
z=None,
|
||||
reserved_frames_mask=None,
|
||||
seq_len=None,
|
||||
guidance=None,
|
||||
is_warmup=batch.is_warmup,
|
||||
cfg_policy=cfg_policy,
|
||||
# DPM-Solver advances once per timestep, irrespective of solver order
|
||||
extra={"progress_step_interval": 1},
|
||||
clip_condition=clip_condition,
|
||||
reference_image_embeddings=reference_image_embeddings,
|
||||
cfg_parallel=cfg_parallel,
|
||||
guidance_scale=guidance_scale,
|
||||
)
|
||||
# clip trajectories are not part of the assembled-video output contract
|
||||
self._run_denoising_loop(ctx, batch, server_args, collect_trajectory=False)
|
||||
return ctx.latents[:, 1:]
|
||||
|
||||
def _before_denoising_loop(
|
||||
self, ctx: WanAnimate2DenoisingContext, batch: Req, server_args: ServerArgs
|
||||
) -> None:
|
||||
# activate the DiT before the reference pass, not just the first denoise step
|
||||
self.begin_declared_component_use(
|
||||
component_name="transformer", module=self.transformer
|
||||
)
|
||||
with set_forward_context(
|
||||
current_timestep=0, attn_metadata=None, forward_batch=batch
|
||||
):
|
||||
ctx.reference_kv = self.transformer.build_reference_kv(
|
||||
clip_cond=ctx.clip_condition
|
||||
)
|
||||
|
||||
def _prepare_step_state(
|
||||
self,
|
||||
ctx: WanAnimate2DenoisingContext,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
step_index: int,
|
||||
t_host: torch.Tensor,
|
||||
timesteps_cpu: torch.Tensor,
|
||||
) -> DenoisingStepState:
|
||||
return DenoisingStepState(
|
||||
step_index=step_index,
|
||||
t_host=t_host,
|
||||
t_device=ctx.timesteps[step_index],
|
||||
t_int=int(t_host.item()),
|
||||
current_model=self.transformer,
|
||||
current_guidance_scale=ctx.guidance_scale,
|
||||
attn_metadata=None,
|
||||
)
|
||||
|
||||
def _run_denoising_step(
|
||||
self,
|
||||
ctx: WanAnimate2DenoisingContext,
|
||||
step: DenoisingStepState,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
) -> None:
|
||||
def predict(branch: CFGBranch) -> torch.Tensor:
|
||||
with set_forward_context(
|
||||
current_timestep=step.step_index,
|
||||
attn_metadata=None,
|
||||
forward_batch=batch,
|
||||
):
|
||||
if branch.name not in ctx.context_by_branch:
|
||||
ctx.context_by_branch[branch.name] = (
|
||||
self.transformer.prepare_context(
|
||||
branch.kwargs["encoder_hidden_states"],
|
||||
ctx.reference_image_embeddings,
|
||||
)
|
||||
)
|
||||
prediction = self.transformer(
|
||||
hidden_states=ctx.latents,
|
||||
timestep=step.t_device.unsqueeze(0),
|
||||
encoder_hidden_states_image=ctx.reference_image_embeddings,
|
||||
clip_cond=ctx.clip_condition,
|
||||
reference_kv=ctx.reference_kv,
|
||||
projected_context=ctx.context_by_branch[branch.name],
|
||||
**branch.kwargs,
|
||||
)
|
||||
return prediction.contiguous() if ctx.cfg_parallel else prediction
|
||||
|
||||
predictions = (
|
||||
run_cfg_parallel(ctx.cfg_policy, predict)
|
||||
if ctx.cfg_parallel
|
||||
else [predict(branch) for branch in ctx.cfg_policy.branches]
|
||||
)
|
||||
noise_pred = predictions[0]
|
||||
if len(predictions) == 2:
|
||||
noise_pred = predictions[1] + step.current_guidance_scale * (
|
||||
noise_pred - predictions[1]
|
||||
)
|
||||
ctx.latents = ctx.scheduler.step(
|
||||
noise_pred,
|
||||
step.t_device,
|
||||
ctx.latents.unsqueeze(0),
|
||||
return_dict=False,
|
||||
)[0].squeeze(0)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
request_state = request_state_from_batch(batch)
|
||||
|
||||
output_frames: list[np.ndarray] = [] # [H, W, C] uint8 frames
|
||||
|
||||
# [C, num_frames_conditioning, H, W] bf16 in [-1, 1]
|
||||
prev_clip_conditioning_frames: torch.Tensor | None = None
|
||||
|
||||
# The DiT runs in bf16 autocast like upstream Wan (timestep and norm math in fp32).
|
||||
with torch.autocast(
|
||||
device_type=current_platform.device_type,
|
||||
dtype=torch.bfloat16,
|
||||
enabled=True,
|
||||
):
|
||||
for clip_denoising_metadata in request_state.schedule:
|
||||
is_clip_0 = clip_denoising_metadata.clip_index == 0
|
||||
|
||||
if not is_clip_0 and prev_clip_conditioning_frames is None:
|
||||
raise ValueError(
|
||||
"For all clips except clip-0 `prev_clip_conditioning_frames` must be not None."
|
||||
)
|
||||
clip_condition = build_clip_conditioning(
|
||||
request_state,
|
||||
clip_denoising_metadata,
|
||||
prev_clip_conditioning_frames,
|
||||
vae_encoder=self._vae_encoder,
|
||||
image_embedder=self._image_embedder,
|
||||
)
|
||||
|
||||
latents = self._denoise_single_clip(
|
||||
batch,
|
||||
server_args,
|
||||
request_state,
|
||||
clip_condition=clip_condition,
|
||||
)
|
||||
|
||||
denoised_latents = latents.to(dtype=torch.float32)
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae.vae
|
||||
):
|
||||
# [1, C, T, H, W] fp32 in [-1, 1]
|
||||
decoded_frames = self.clip_decoder.decode_raw(
|
||||
denoised_latents.unsqueeze(0),
|
||||
server_args,
|
||||
vae_dtype=torch.float32,
|
||||
)
|
||||
|
||||
# [T, H, W, C] uint8
|
||||
output_frames_for_clip = (
|
||||
rearrange(((decoded_frames + 1) * 127.5), "1 c t h w -> t h w c")
|
||||
.detach()
|
||||
.to(torch.uint8)
|
||||
.cpu()
|
||||
.numpy()
|
||||
)
|
||||
|
||||
# Drop the overlap frames the previous clip already emitted; 0 for clip 0.
|
||||
output_frames_for_clip = output_frames_for_clip[
|
||||
clip_denoising_metadata.num_frames_conditioning_for_clip :
|
||||
]
|
||||
output_frames.extend(output_frames_for_clip)
|
||||
|
||||
# The next clip takes the last ``num_frames_conditioning`` frames.
|
||||
prev_clip_conditioning_frames = (
|
||||
decoded_frames[
|
||||
0, :, -request_state.inputs.num_frames_conditioning :
|
||||
]
|
||||
.detach()
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
# only the independent overlap frames survive into the next clip
|
||||
del decoded_frames, denoised_latents, latents, clip_condition
|
||||
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
# Already-decoded [T, H, W, C] uint8; WanAnimate2OutputStage must not re-decode.
|
||||
request_state.decoded_frames = self._post_process_frames(
|
||||
output_frames,
|
||||
request_state.num_reference_video_frames,
|
||||
request_state.letterbox_info,
|
||||
)
|
||||
return batch
|
||||
|
||||
@staticmethod
|
||||
def _post_process_frames(
|
||||
frames: list[np.ndarray],
|
||||
num_output_frames: int,
|
||||
letterbox_info: LetterboxInfo,
|
||||
) -> np.ndarray:
|
||||
"""Stack ``list[np.ndarray]`` to ``[T, H, W, C]`` uint8, trim to ``num_output_frames`` and crop the
|
||||
``resize_by_area`` letterbox."""
|
||||
if len(frames) == 0:
|
||||
raise ValueError(
|
||||
"WanAnimate2DenoisingStage produced 0 decoded frames to post-process."
|
||||
)
|
||||
frames_np = np.stack(frames, axis=0) # [T, H, W, C]
|
||||
frames_np = frames_np[:num_output_frames]
|
||||
|
||||
return letterbox_info.crop(frames_np)
|
||||
|
||||
|
||||
class WanAnimate2OutputStage(PipelineStage):
|
||||
"""Terminal stage: wrap the already-decoded, assembled frames in an OutputBatch.
|
||||
|
||||
Replaces the standard DecodingStage (which would VAE-decode a single-clip latent).
|
||||
"""
|
||||
|
||||
@property
|
||||
def role_affinity(self) -> RoleType:
|
||||
# The terminal slot of the standard stage layout; disaggregated deployment itself
|
||||
# is rejected by Wan_Animate_2_14B_Config.supports_disaggregation.
|
||||
return RoleType.DECODER
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
|
||||
result.add_check(
|
||||
"decoded_frames",
|
||||
(
|
||||
request_state.decoded_frames
|
||||
if isinstance(request_state, WanAnimate2RequestState)
|
||||
else None
|
||||
),
|
||||
_is_decoded_frames,
|
||||
)
|
||||
return result
|
||||
|
||||
def verify_output(
|
||||
self, batch: OutputBatch, server_args: ServerArgs
|
||||
) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("output", batch.output, V.list_not_empty)
|
||||
return result
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
request_state = request_state_from_batch(batch)
|
||||
frames = request_state.decoded_frames
|
||||
if frames is None:
|
||||
raise ValueError(
|
||||
"WanAnimate2OutputStage: WanAnimate2DenoisingStage did not assemble decoded frames."
|
||||
)
|
||||
# frames: [T, H, W, C] uint8, consumed directly by the save path. Audio, when
|
||||
# extracted, is [1, C, L] fp32 in [-1, 1] with its sample rate; None gives a silent mp4.
|
||||
return OutputBatch(
|
||||
output=[frames],
|
||||
audio=request_state.audio,
|
||||
audio_sample_rate=request_state.audio_sample_rate,
|
||||
metrics=batch.metrics,
|
||||
)
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Adapters exposing the official Wan encoder API over the diffusers/transformers models the
|
||||
# sglang loaders produce: image_encoder.visual takes list[torch.Tensor] of [C, T, H, W] and returns
|
||||
# [B, 257, 1280]; vae.encode/decode fold in the Wan latent scaling.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import CLIPImageProcessor
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
|
||||
|
||||
class WanAnimate2ImageEncoderAdapter:
|
||||
"""Official Wan CLIP API over a ``CLIPVisionModel``: ``visual`` takes ``list[torch.Tensor]`` of ``[C, T, H, W]``
|
||||
in [-1, 1] and returns ``hidden_states[-2]`` as ``[B, 257, 1280]`` in the model dtype, the pre-final-LayerNorm
|
||||
slice the Wan-I2V ``postprocess_image`` consumes."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: nn.Module,
|
||||
image_size: int,
|
||||
image_mean: tuple[float, float, float],
|
||||
image_std: tuple[float, float, float],
|
||||
hidden_state_index: int = -2,
|
||||
device: torch.device | str | None = None,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.image_size = image_size
|
||||
self.image_mean = image_mean
|
||||
self.image_std = image_std
|
||||
self.hidden_state_index = hidden_state_index
|
||||
self.device = device
|
||||
|
||||
@classmethod
|
||||
def from_loaded(
|
||||
cls,
|
||||
model: nn.Module,
|
||||
image_processor: CLIPImageProcessor,
|
||||
device: torch.device | str | None = None,
|
||||
) -> WanAnimate2ImageEncoderAdapter:
|
||||
"""Build the adapter with image_size / mean / std from the checkpoint's
|
||||
``CLIPImageProcessor`` (``crop_size`` is the post-resize crop the model sees)."""
|
||||
crop_h = int(image_processor.crop_size["height"])
|
||||
crop_w = int(image_processor.crop_size["width"])
|
||||
if crop_h != crop_w:
|
||||
raise ValueError(
|
||||
"WanAnimate2ImageEncoderAdapter expects a square CLIP crop_size; got "
|
||||
f"height={crop_h}, width={crop_w}."
|
||||
)
|
||||
return cls(
|
||||
model,
|
||||
image_size=crop_h,
|
||||
image_mean=tuple(image_processor.image_mean),
|
||||
image_std=tuple(image_processor.image_std),
|
||||
device=device,
|
||||
)
|
||||
|
||||
def _resolve_device(self) -> torch.device:
|
||||
if self.device is not None:
|
||||
return torch.device(self.device)
|
||||
return next(self.model.parameters()).device
|
||||
|
||||
@torch.no_grad()
|
||||
def visual(self, videos: list[torch.Tensor]) -> torch.Tensor:
|
||||
# The caller holds the component through the residency manager, so the model is
|
||||
# already where it should be; inputs follow it. Runs in the model's own dtype with
|
||||
# autocast off: the fused patch-embed conv does not autocast an fp32 weight against
|
||||
# a bf16 input.
|
||||
device = self._resolve_device()
|
||||
model_dtype = next(self.model.parameters()).dtype
|
||||
size = (self.image_size, self.image_size)
|
||||
|
||||
# Each video [C, T, H, W] becomes T images [T, C, image_size, image_size]; the images
|
||||
# of all videos are stacked into one batch [N, C, image_size, image_size], N = total
|
||||
# frame count.
|
||||
pixels = torch.cat(
|
||||
[
|
||||
F.interpolate(
|
||||
video.transpose(0, 1).to(device=device, dtype=model_dtype),
|
||||
size=size,
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
)
|
||||
for video in videos
|
||||
]
|
||||
)
|
||||
|
||||
# [-1, 1] -> [0, 1] then Normalize(mean, std), in the model's own dtype.
|
||||
mean = torch.tensor(self.image_mean, device=device, dtype=pixels.dtype).view(
|
||||
1, 3, 1, 1
|
||||
)
|
||||
std = torch.tensor(self.image_std, device=device, dtype=pixels.dtype).view(
|
||||
1, 3, 1, 1
|
||||
)
|
||||
pixels = pixels.mul(0.5).add(0.5).sub(mean).div(std)
|
||||
|
||||
# Encode; return the hidden_states[-2] slice.
|
||||
with (
|
||||
torch.autocast(device_type=device.type, enabled=False),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
outputs = self.model(pixel_values=pixels, output_hidden_states=True)
|
||||
return outputs.hidden_states[self.hidden_state_index]
|
||||
|
||||
|
||||
class WanAnimate2VaeAdapter(nn.Module):
|
||||
"""Stage-local latent-scaling adapter over the pipeline's native ``AutoencoderKLWan``.
|
||||
``encode`` takes ``list[torch.Tensor]`` (or ``[B, C, T, H, W]``) and returns ``list[torch.Tensor]`` of
|
||||
``[16, T', H', W']`` fp32, ``(mode() - latents_mean) / latents_std``; ``decode`` inverts that and returns
|
||||
``[B, C, T, H, W]`` fp32 in [-1, 1], which is why ``Wan_Animate_2_14B_Config.get_decode_scale_and_shift``
|
||||
returns ``(1.0, None)``."""
|
||||
|
||||
@property
|
||||
def use_parallel_decode(self) -> bool:
|
||||
return self.vae.use_parallel_decode
|
||||
|
||||
def enable_tiling(self) -> None:
|
||||
self.vae.enable_tiling()
|
||||
|
||||
def __init__(self, vae: nn.Module) -> None:
|
||||
super().__init__()
|
||||
if not isinstance(vae, nn.Module):
|
||||
raise TypeError(
|
||||
"WanAnimate2VaeAdapter expects an nn.Module AutoencoderKLWan; got "
|
||||
f"{type(vae).__name__!r}."
|
||||
)
|
||||
self.vae = vae
|
||||
|
||||
# Per-channel latent scaling (length z_dim), which the sglang AutoencoderKLWan
|
||||
# copies from its config onto the module. Plain (unregistered) tensors, reshaped
|
||||
# to broadcast over [B, C, T, H, W].
|
||||
self._latents_mean = torch.as_tensor(
|
||||
vae.latents_mean, dtype=torch.float32
|
||||
).view(1, -1, 1, 1, 1)
|
||||
self._latents_std = torch.as_tensor(vae.latents_std, dtype=torch.float32).view(
|
||||
1, -1, 1, 1, 1
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _to_list_of_tensors(
|
||||
items: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
|
||||
) -> list[torch.Tensor]:
|
||||
"""Per-item tensors from either a list or a batched ``[B, ...]`` tensor; the stages
|
||||
pass lists, the shared decode path passes a batch."""
|
||||
if isinstance(items, (list, tuple)):
|
||||
return list(items)
|
||||
return [items[i] for i in range(items.shape[0])]
|
||||
|
||||
def _mean_std_like(self, input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
self._latents_mean.to(device=input.device, dtype=input.dtype),
|
||||
self._latents_std.to(device=input.device, dtype=input.dtype),
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def encode(
|
||||
self,
|
||||
videos: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> list[torch.Tensor]:
|
||||
out: list[torch.Tensor] = []
|
||||
for sample in self._to_list_of_tensors(videos):
|
||||
# Run fp32 with autocast off: the fused causal-conv path has no dtype guard, so
|
||||
# an outer autocast(bf16) would cast the conv input against the fp32 bias.
|
||||
# The sglang AutoencoderKLWan returns the DiagonalGaussianDistribution itself,
|
||||
# not a diffusers AutoencoderKLOutput.
|
||||
with torch.autocast(device_type=sample.device.type, enabled=False):
|
||||
posterior = self.vae.encode(sample.unsqueeze(0).float())
|
||||
|
||||
# Deterministic mu (not a sampled draw).
|
||||
mu = posterior.mode().float()
|
||||
mean, std = self._mean_std_like(mu)
|
||||
z = (mu - mean) / std
|
||||
out.append(z.squeeze(0))
|
||||
return out
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(
|
||||
self,
|
||||
latents: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
out: list[torch.Tensor] = []
|
||||
for sample in self._to_list_of_tensors(latents):
|
||||
z = sample.unsqueeze(0).float()
|
||||
mean, std = self._mean_std_like(z)
|
||||
z = z * std + mean
|
||||
|
||||
# fp32, autocast OFF: see ``encode`` above. decode returns a plain tensor.
|
||||
with torch.autocast(device_type=z.device.type, enabled=False):
|
||||
decoded = self.vae.decode(z)
|
||||
out.append(decoded.float().clamp_(-1.0, 1.0).squeeze(0))
|
||||
|
||||
# Clips are decoded individually; avoid copying their full pixel tensor.
|
||||
return out[0].unsqueeze(0) if len(out) == 1 else torch.stack(out, dim=0)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
z: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
return self.decode(z, *args, **kwargs)
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from the Wan-Animate-2 reference implementation (Apache-2.0):
|
||||
# https://github.com/Wan-Video/Wan-Animate-2
|
||||
#
|
||||
# Keep the original Wan preprocessing numerics; the newer Diffusers modular
|
||||
# processor is not a numerically interchangeable replacement. The
|
||||
# zigzag_padding / resize_by_area names are kept for parity with the official helpers;
|
||||
# make_conditioning_mask is the official get_i2v_mask.
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from copy import deepcopy
|
||||
from io import BytesIO
|
||||
from typing import Literal
|
||||
|
||||
import cv2
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.utils.common import get_video_bytes
|
||||
|
||||
__all__ = [
|
||||
"LetterboxInfo",
|
||||
"MIN_SINGLE_CLIP_LEN",
|
||||
"resize_by_area",
|
||||
"letterbox_resize",
|
||||
"get_frame_indices",
|
||||
"get_padding_len",
|
||||
"single_clip_reference_video_len",
|
||||
"zigzag_padding",
|
||||
"make_conditioning_mask",
|
||||
"read_reference_video_frames",
|
||||
"write_reference_video",
|
||||
]
|
||||
|
||||
|
||||
class LetterboxInfo(msgspec.Struct, frozen=True):
|
||||
"""Where letterbox_resize put the content inside the padded canvas."""
|
||||
|
||||
# Axis the padding bars sit on: "width" pads left/right, "height" pads top/bottom.
|
||||
pad_axis: Literal["width", "height"]
|
||||
# Bar size in pixels before the content along pad_axis.
|
||||
padding: int
|
||||
# Content extent in pixels along pad_axis.
|
||||
content_len: int
|
||||
|
||||
def crop(self, frames: np.ndarray) -> np.ndarray:
|
||||
"""Cut the bars back out of ``[T, H, W, C]`` frames."""
|
||||
start, stop = self.padding, self.padding + self.content_len
|
||||
if self.pad_axis == "width":
|
||||
return frames[:, :, start:stop, :]
|
||||
return frames[:, start:stop, :, :]
|
||||
|
||||
|
||||
def resize_by_area(
|
||||
image: np.ndarray,
|
||||
target_area: int,
|
||||
keep_aspect_ratio: bool = True,
|
||||
divisor: int = 16,
|
||||
padding_color: tuple[int, int, int] = (0, 0, 0),
|
||||
) -> tuple[np.ndarray, LetterboxInfo]:
|
||||
"""Resize ``image`` (``[H, W, C]`` uint8) so its area is <= ``target_area``, then letterbox-pad to that size."""
|
||||
h, w = image.shape[:2]
|
||||
aspect_ratio = w / h
|
||||
if keep_aspect_ratio:
|
||||
height_target = math.sqrt(target_area / aspect_ratio)
|
||||
width_target = target_area / height_target
|
||||
else:
|
||||
width_target = height_target = math.sqrt(target_area)
|
||||
width_target, height_target = (
|
||||
int((width_target // divisor) * divisor),
|
||||
int((height_target // divisor) * divisor),
|
||||
)
|
||||
|
||||
# Resize (INTER_AREA when shrinking, INTER_LINEAR when enlarging)
|
||||
# Taken as is from the reference implementation.
|
||||
interpolation = (
|
||||
cv2.INTER_AREA if (width_target * height_target < w * h) else cv2.INTER_LINEAR
|
||||
)
|
||||
|
||||
return letterbox_resize(
|
||||
image,
|
||||
height=height_target,
|
||||
width=width_target,
|
||||
padding_color=padding_color,
|
||||
interpolation=interpolation,
|
||||
)
|
||||
|
||||
|
||||
def letterbox_resize(
|
||||
image: np.ndarray,
|
||||
height: int = 512,
|
||||
width: int = 512,
|
||||
padding_color: tuple[int, int, int] = (0, 0, 0),
|
||||
interpolation: int = cv2.INTER_LINEAR,
|
||||
) -> tuple[np.ndarray, LetterboxInfo]:
|
||||
"""Aspect-preserving resize of ``image`` into a ``[height, width, C]`` uint8 canvas with letterbox padding."""
|
||||
height_source = image.shape[0]
|
||||
width_source = image.shape[1]
|
||||
num_channels_source = image.shape[2]
|
||||
|
||||
canvas = np.zeros((height, width, num_channels_source))
|
||||
if num_channels_source == 1:
|
||||
canvas[:, :, 0] = padding_color[0]
|
||||
else:
|
||||
canvas[:, :, 0] = padding_color[0]
|
||||
canvas[:, :, 1] = padding_color[1]
|
||||
canvas[:, :, 2] = padding_color[2]
|
||||
|
||||
if (height_source / width_source) > (height / width):
|
||||
width_target = int(height / height_source * width_source)
|
||||
img = cv2.resize(image, (width_target, height), interpolation=interpolation)
|
||||
padding = int((width - width_target) / 2)
|
||||
if len(img.shape) == 2:
|
||||
img = img[
|
||||
:, :, np.newaxis
|
||||
] # grayscale image: add a num_channels_source dimension
|
||||
canvas[:, padding : padding + width_target, :] = img
|
||||
pad_axis = "width"
|
||||
content_len = width_target
|
||||
else:
|
||||
height_target = int(width / width_source * height_source)
|
||||
img = cv2.resize(image, (width, height_target), interpolation=interpolation)
|
||||
padding = int((height - height_target) / 2)
|
||||
if len(img.shape) == 2:
|
||||
img = img[
|
||||
:, :, np.newaxis
|
||||
] # grayscale image: add a num_channels_source dimension
|
||||
canvas[padding : padding + height_target, :, :] = img
|
||||
pad_axis = "height"
|
||||
content_len = height_target
|
||||
|
||||
canvas = np.uint8(canvas)
|
||||
return canvas, LetterboxInfo(
|
||||
pad_axis=pad_axis, padding=padding, content_len=content_len
|
||||
)
|
||||
|
||||
|
||||
def get_frame_indices(
|
||||
num_frames_source: int, fps_source: float, num_frames_target: int, fps_target: float
|
||||
) -> list[int]:
|
||||
"""Source frame index (``list[int]``, nearest frame clipped to the source range) for each of the
|
||||
``num_frames_target`` output frames when resampling from ``fps_source`` to ``fps_target``."""
|
||||
|
||||
times = np.arange(0, num_frames_target) / fps_target
|
||||
frame_indices_source = np.round(times * fps_source).astype(int)
|
||||
frame_indices_source = np.clip(frame_indices_source, 0, num_frames_source - 1)
|
||||
|
||||
return frame_indices_source.tolist()
|
||||
|
||||
|
||||
# Fewest frames the last clip may add beyond its overlap; a multiple of 4 so it also sits
|
||||
# on the temporal-VAE grid. Taken as is from the reference.
|
||||
_MIN_LAST_CLIP_NEW_FRAMES = 28
|
||||
|
||||
|
||||
def get_padding_len(
|
||||
num_frames: int, clip_len: int, num_frames_conditioning: int = 1
|
||||
) -> int:
|
||||
"""Reference-video length after zigzag padding so the last clip is long enough and
|
||||
4k-aligned; taken as is from the reference implementation."""
|
||||
|
||||
remaining = (num_frames - num_frames_conditioning) % (
|
||||
clip_len - num_frames_conditioning
|
||||
)
|
||||
# Short leftover: raise it to the minimum. Otherwise: round up to the next multiple
|
||||
# of 4 (VAE temporal stride); the minimum is itself a multiple of 4.
|
||||
if remaining < _MIN_LAST_CLIP_NEW_FRAMES:
|
||||
padding_needed = _MIN_LAST_CLIP_NEW_FRAMES - remaining
|
||||
else:
|
||||
padding_needed = 4 - remaining % 4
|
||||
|
||||
return num_frames + padding_needed
|
||||
|
||||
|
||||
# get_padding_len extends every reference video to at least this many frames, so a shorter
|
||||
# clip_len can never hold the whole video in one clip.
|
||||
MIN_SINGLE_CLIP_LEN = _MIN_LAST_CLIP_NEW_FRAMES + 1
|
||||
|
||||
|
||||
def single_clip_reference_video_len(
|
||||
clip_len: int, num_frames_conditioning: int = 1
|
||||
) -> int:
|
||||
"""Reference-video frame count that get_padding_len extends to exactly ``clip_len``,
|
||||
so build_schedule yields one full-length clip; used to size the synthetic warmup video."""
|
||||
num_frames = clip_len - 1
|
||||
if (
|
||||
clip_len < MIN_SINGLE_CLIP_LEN
|
||||
or get_padding_len(num_frames, clip_len, num_frames_conditioning) != clip_len
|
||||
):
|
||||
raise ValueError(
|
||||
f"clip_len={clip_len} cannot hold a reference video in a single clip; "
|
||||
f"the schedule needs clip_len >= {MIN_SINGLE_CLIP_LEN}"
|
||||
)
|
||||
return num_frames
|
||||
|
||||
|
||||
def zigzag_padding(array: list, target_len: int) -> list:
|
||||
"""Pad ``array`` up to ``target_len`` by bouncing the read index back and forth (deep copies)."""
|
||||
if target_len < len(array):
|
||||
raise ValueError(
|
||||
f"zigzag_padding only extends: target_len {target_len} < len(array) {len(array)}"
|
||||
)
|
||||
index = 0
|
||||
flip = False
|
||||
target_array = []
|
||||
while len(target_array) < target_len:
|
||||
target_array.append(deepcopy(array[index]))
|
||||
if flip:
|
||||
index -= 1
|
||||
else:
|
||||
index += 1
|
||||
if index == 0 or index == len(array) - 1:
|
||||
flip = not flip
|
||||
return target_array
|
||||
|
||||
|
||||
def make_conditioning_mask(
|
||||
latent_t: int,
|
||||
latent_h: int,
|
||||
latent_w: int,
|
||||
num_prefix_conditioning_frames: int = 1,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> torch.Tensor:
|
||||
"""Build the ``[4, latent_t, latent_h, latent_w]`` fp32 i2v conditioning mask; ``num_prefix_conditioning_frames`` is the
|
||||
conditioned prefix length in pixel frames."""
|
||||
|
||||
# The VAE maps the first frame to 1 frame in latent space.
|
||||
# For each subsequent 4 pixel frames it maps them to 1 frame in the latent space.
|
||||
# Hence, the ``(latent_t - 1) * 4 + 1``.
|
||||
# VAE: 4k + 1 pixel frames -> k + 1 latent frames
|
||||
mask = torch.zeros(1, (latent_t - 1) * 4 + 1, latent_h, latent_w, device=device)
|
||||
|
||||
mask[:, :num_prefix_conditioning_frames] = 1
|
||||
mask = torch.concat(
|
||||
[torch.repeat_interleave(mask[:, 0:1], repeats=4, dim=1), mask[:, 1:]], dim=1
|
||||
) # repeat the first pixel frame's flag 4x: (k - 1) * 4 + 1 -> 4 * k pixel slots
|
||||
|
||||
mask = mask.view(1, mask.shape[1] // 4, 4, latent_h, latent_w)
|
||||
# (1, 4*(k + 1), latent_h, latent_w) -> (1, (k + 1), 4, latent_h, latent_w)
|
||||
|
||||
mask = mask.transpose(1, 2)[0]
|
||||
# (1, (k + 1), 4, latent_h, latent_w) -> (4, (k + 1), latent_h, latent_w)
|
||||
return mask
|
||||
|
||||
|
||||
def read_reference_video_frames(path: str, fps_target: int) -> np.ndarray:
|
||||
"""Decode the reference video resampled to ``fps_target``; returns ``[N, H, W, 3]`` uint8 RGB.
|
||||
|
||||
Three readers: decord on CPU as the reference implementation does (its GPU decoder
|
||||
exists only in CUDA builds); decord2 is the arm64 build of the same API; PyAV is the
|
||||
fallback when neither is installed.
|
||||
"""
|
||||
source = (
|
||||
BytesIO(get_video_bytes(path))
|
||||
if path.startswith(("http://", "https://", "data:"))
|
||||
else path
|
||||
)
|
||||
try:
|
||||
from decord import VideoReader
|
||||
except ImportError:
|
||||
try:
|
||||
from decord2 import VideoReader
|
||||
except ImportError:
|
||||
VideoReader = None
|
||||
|
||||
if VideoReader is not None:
|
||||
video_reader = VideoReader(source)
|
||||
num_frames_source = len(video_reader)
|
||||
fps_source = video_reader.get_avg_fps()
|
||||
num_frames_target = int(num_frames_source / fps_source * fps_target)
|
||||
indices = get_frame_indices(
|
||||
num_frames_source, fps_source, num_frames_target, fps_target
|
||||
)
|
||||
return video_reader.get_batch(indices).asnumpy()
|
||||
|
||||
import av
|
||||
|
||||
# Decode everything once so num_frames_source matches decord's len() exactly.
|
||||
with av.open(source) as container:
|
||||
stream = container.streams.video[0]
|
||||
fps_source = float(stream.average_rate)
|
||||
frames_source = [
|
||||
frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)
|
||||
]
|
||||
num_frames_source = len(frames_source)
|
||||
num_frames_target = int(num_frames_source / fps_source * fps_target)
|
||||
indices = get_frame_indices(
|
||||
num_frames_source, fps_source, num_frames_target, fps_target
|
||||
)
|
||||
return np.stack([frames_source[i] for i in indices])
|
||||
|
||||
|
||||
def write_reference_video(path: str, frames: np.ndarray, fps: int) -> None:
|
||||
"""Write ``[N, H, W, 3]`` uint8 RGB frames to ``path`` as an mp4 at ``fps`` through the
|
||||
runtime's video writer (the one request outputs go through), so the synthetic warmup
|
||||
video is encoded exactly like a request output and decodes with the readers above."""
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import post_process_sample
|
||||
|
||||
try:
|
||||
post_process_sample(frames, DataType.VIDEO, fps, save_file_path=path)
|
||||
except Exception as e:
|
||||
raise RuntimeError(
|
||||
f"the shared video writer (post_process_sample) could not write the "
|
||||
f"synthetic reference video {path}: {e}"
|
||||
) from e
|
||||
|
||||
|
||||
def validate_and_get_single_string(value: str | list[str] | None, name: str) -> str:
|
||||
"""This pipeline produces one video per request, so a batched prompt field,
|
||||
video path or image path must hold exactly one string."""
|
||||
if isinstance(value, list):
|
||||
if len(value) != 1:
|
||||
raise ValueError(
|
||||
f"Wan-Animate-2 takes exactly one '{name}' per request, got {len(value)}"
|
||||
)
|
||||
value = value[0]
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(
|
||||
f"Wan-Animate-2 requires '{name}' to be a string, got {value!r}"
|
||||
)
|
||||
return value
|
||||
@@ -270,6 +270,18 @@
|
||||
"ssim_threshold": 0.48,
|
||||
"psnr_threshold": 13.0,
|
||||
"mean_abs_diff_threshold": 45.0
|
||||
},
|
||||
"wan2_2_animate_2_14b_ref2v": {
|
||||
"clip_threshold": 0.90,
|
||||
"ssim_threshold": 0.92,
|
||||
"psnr_threshold": 24.0,
|
||||
"mean_abs_diff_threshold": 10.0
|
||||
},
|
||||
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
|
||||
"clip_threshold": 0.90,
|
||||
"ssim_threshold": 0.92,
|
||||
"psnr_threshold": 24.0,
|
||||
"mean_abs_diff_threshold": 10.0
|
||||
}
|
||||
},
|
||||
"default_clip_threshold_image": 0.92,
|
||||
|
||||
@@ -39,6 +39,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
|
||||
SANA_WM_TI2V_CI_sampling_params,
|
||||
T2I_sampling_params,
|
||||
T2V_sampling_params,
|
||||
WAN_2_2_ANIMATE_2_14B_sampling_params,
|
||||
_make_modelopt_ci_case,
|
||||
_with_default_num_gpus,
|
||||
)
|
||||
@@ -63,6 +64,7 @@ from sglang.multimodal_gen.test.test_utils import (
|
||||
DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
|
||||
DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST,
|
||||
@@ -459,6 +461,23 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
run_models_api_check=False,
|
||||
run_t2v_input_reference_check=False,
|
||||
),
|
||||
# Reference image + driving video -> video. The DiT peaks at 76 GB resident,
|
||||
# so the 80 GB lane streams it layerwise; this also covers the uncond block-9 skip.
|
||||
DiffusionTestCase(
|
||||
"wan2_2_animate_2_14b_ref2v",
|
||||
DiffusionServerArgs(
|
||||
model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
|
||||
modality="video",
|
||||
dit_layerwise_offload=True,
|
||||
text_encoder_cpu_offload=True,
|
||||
extras=["--layerwise-offload-components", "transformer"],
|
||||
),
|
||||
WAN_2_2_ANIMATE_2_14B_sampling_params,
|
||||
# The DiT forward takes a per-clip conditioning container and runs forward_ref,
|
||||
# so the raw-component harness contract does not apply.
|
||||
run_perf_check=True,
|
||||
run_component_accuracy_check=False,
|
||||
),
|
||||
# flaky
|
||||
# === Helios T2V ===
|
||||
# DiffusionTestCase(
|
||||
@@ -975,6 +994,32 @@ TWO_GPU_CASES = [
|
||||
model_path=DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
|
||||
),
|
||||
),
|
||||
# GT = the 1-GPU frames renamed. With replicated encoders, serial VAE decode and the
|
||||
# VAE channels_last_3d layout the loader otherwise applies only for num_gpus == 1,
|
||||
# CFG-parallel output is bit-identical to the 1-GPU case, so both share one ground truth.
|
||||
DiffusionTestCase(
|
||||
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu",
|
||||
DiffusionServerArgs(
|
||||
model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
|
||||
modality="video",
|
||||
cfg_parallel=True,
|
||||
# On 80 GB cards the resident DiT leaves no room for the full-shape warmup probe; stream it layerwise (bit-identical output).
|
||||
dit_layerwise_offload=True,
|
||||
text_encoder_cpu_offload=True,
|
||||
extras=[
|
||||
"--layerwise-offload-components",
|
||||
"transformer",
|
||||
"--encoder-parallel",
|
||||
"replicate",
|
||||
"--vae-config.use-parallel-decode",
|
||||
"false",
|
||||
],
|
||||
env_vars={"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "1"},
|
||||
),
|
||||
WAN_2_2_ANIMATE_2_14B_sampling_params,
|
||||
run_perf_check=True,
|
||||
run_component_accuracy_check=False,
|
||||
),
|
||||
DiffusionTestCase(
|
||||
"wan2_2_t2v_a14b_2gpu",
|
||||
DiffusionServerArgs(
|
||||
|
||||
@@ -3524,6 +3524,60 @@
|
||||
"expected_avg_denoise_ms": 0.0,
|
||||
"expected_median_denoise_ms": 0.0,
|
||||
"estimated_full_test_time_s": 73.7
|
||||
},
|
||||
"wan2_2_animate_2_14b_ref2v": {
|
||||
"stages_ms": {
|
||||
"WanAnimate2BeforeDenoisingStage": 3783.78,
|
||||
"WanAnimate2DenoisingStage": 94604.96,
|
||||
"WanAnimate2OutputStage": 0.06
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 25832.83,
|
||||
"1": 7210.15,
|
||||
"2": 11187.28,
|
||||
"3": 8460.7,
|
||||
"4": 8407.93,
|
||||
"5": 8409.07,
|
||||
"6": 8408.07,
|
||||
"7": 8415.78
|
||||
},
|
||||
"expected_e2e_ms": 98402.77,
|
||||
"expected_load_ms": 62901.15,
|
||||
"expected_avg_denoise_ms": 10791.48,
|
||||
"expected_median_denoise_ms": 8412.42,
|
||||
"load_peak_vram_mb": 1776.0,
|
||||
"runtime_peak_vram_mb": 36322.0,
|
||||
"load_peak_allocated_mb": 1737.0,
|
||||
"runtime_peak_allocated_mb": 25748.0,
|
||||
"estimated_full_test_time_s": 489.9,
|
||||
"warmup_peak_vram_mb": 14598.0
|
||||
},
|
||||
"wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
|
||||
"stages_ms": {
|
||||
"WanAnimate2BeforeDenoisingStage": 5778.62,
|
||||
"WanAnimate2DenoisingStage": 58196.97,
|
||||
"WanAnimate2OutputStage": 0.06
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 21723.06,
|
||||
"1": 3125.83,
|
||||
"2": 4335.41,
|
||||
"3": 4337.63,
|
||||
"4": 4347.53,
|
||||
"5": 4339.76,
|
||||
"6": 4350.68,
|
||||
"7": 4280.35
|
||||
},
|
||||
"expected_e2e_ms": 63988.59,
|
||||
"expected_load_ms": 64309.0,
|
||||
"expected_avg_denoise_ms": 6355.03,
|
||||
"expected_median_denoise_ms": 4338.7,
|
||||
"load_peak_vram_mb": 1776.0,
|
||||
"runtime_peak_vram_mb": 36322.0,
|
||||
"load_peak_allocated_mb": 1737.0,
|
||||
"runtime_peak_allocated_mb": 25745.0,
|
||||
"estimated_full_test_time_s": 404.0,
|
||||
"warmup_peak_vram_mb": 14598.0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,10 @@ from sglang.multimodal_gen.registry import (
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
|
||||
from sglang.multimodal_gen.test.test_utils import (
|
||||
SGL_TEST_FILES_CI_DATA_REPO,
|
||||
SGL_TEST_FILES_CI_DATA_REVISION,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -885,6 +889,35 @@ TURBOWAN_I2V_sampling_params = DiffusionSamplingParams(
|
||||
fps=4,
|
||||
)
|
||||
|
||||
# Wan-Animate-2 inputs; they ship in the same ci-data-diffusion commit as the GT,
|
||||
# so the pinned revision must point at a commit that carries them.
|
||||
_WAN_2_2_ANIMATE_2_CI_INPUT_BASE = (
|
||||
"https://raw.githubusercontent.com/"
|
||||
f"{SGL_TEST_FILES_CI_DATA_REPO}/{SGL_TEST_FILES_CI_DATA_REVISION}/"
|
||||
"diffusion-ci/inputs/wan_animate_2"
|
||||
)
|
||||
WAN_2_2_ANIMATE_2_14B_sampling_params = DiffusionSamplingParams(
|
||||
prompt=(
|
||||
"Character appearance description: a smiling young woman with shoulder-length "
|
||||
"dark hair, a mustard-yellow knit sweater, blue jeans and white sneakers. "
|
||||
"Background description: a plain light-gray studio backdrop."
|
||||
),
|
||||
image_path=f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/reference_short.jpg",
|
||||
# The model reads `size` as an area budget and emits the 16-aligned canvas fitted to
|
||||
# the reference aspect; 576x1024 (9:16, like the image) maps onto itself.
|
||||
output_size="576x1024",
|
||||
fps=16,
|
||||
output_format="mp4",
|
||||
# 24 driving frames at 16 fps pad to one 29-frame clip (get_padding_len),
|
||||
# so the request denoises a single clip; output length follows the driving video.
|
||||
extras={
|
||||
"video_url": f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/driving_video_short.mp4",
|
||||
"clip_len": 37,
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
},
|
||||
)
|
||||
|
||||
HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams(
|
||||
prompt="",
|
||||
image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png",
|
||||
|
||||
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
|
||||
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
|
||||
# publish.
|
||||
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "17312c8df37e65bffe75818884bc2a5c93cf3094"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "31d8ad69f2434727aad518f0678fbd57823856fe"
|
||||
|
||||
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
|
||||
# when it's regenerated on its own cadence.
|
||||
@@ -200,6 +200,9 @@ DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST = (
|
||||
DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
|
||||
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
|
||||
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
|
||||
DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST = (
|
||||
"Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
|
||||
)
|
||||
|
||||
# MOVA video generation models
|
||||
DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST = "OpenMOSS-Team/MOVA-360p"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,799 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Unit tests for the Wan-Animate-2 block mask, shape math and per-clip reference K/V ownership.
|
||||
|
||||
Single-process and CPU only, in the style of ``test_sp_shard.py`` / ``test_usp_ring_tail_pad.py``:
|
||||
``_get_block_mask_from_layout`` is called unbound with a stub ``self`` and patched module globals; the
|
||||
shape helpers are called directly. The parallelism rejections live in ``test_wan_animate_2_config.py``.
|
||||
"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits import wan_animate_2_block
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import (
|
||||
WanAnimate2Transformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
|
||||
WanAnimate2TransformerBlock,
|
||||
_apply_rope_interleaved,
|
||||
_InContextLayout,
|
||||
_make_score_mod,
|
||||
_sp_scatter_out,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
|
||||
WanAnimate2ClipConditioning,
|
||||
WanAnimate2ReferenceKV,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.wanvideo import (
|
||||
WanTransformer3DModel,
|
||||
WanTransformerBlock,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
|
||||
WanAnimate2DenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
ensure_distributed_env_defaults,
|
||||
)
|
||||
|
||||
_DIT = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2"
|
||||
_BLOCK = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block"
|
||||
_DENOISING = (
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages"
|
||||
".wan_animate_2.denoising"
|
||||
)
|
||||
|
||||
|
||||
# A short last clip against the full-length clip the mask is built for: this clip is
|
||||
# (2, 1, 2) = 2 frames x 2 tokens, the reference video (1, 1, 2), the full clip (4, 2, 2) =
|
||||
# 4 frames x 4 tokens. Every mask-side field (max_*) differs from the clip's own, so a mask
|
||||
# built from the clip's grid, or attention padding that disagrees with the mask, fails below.
|
||||
_SHORT_CLIP_LAYOUT = _InContextLayout((2, 1, 2), (1, 1, 2), (4, 2, 2))
|
||||
|
||||
|
||||
def _mask_probe(mask_fn):
|
||||
def valid(q_idx, kv_idx):
|
||||
out = mask_fn(
|
||||
torch.tensor(0),
|
||||
torch.tensor(0),
|
||||
torch.tensor(q_idx),
|
||||
torch.tensor(kv_idx),
|
||||
)
|
||||
return bool(out.item())
|
||||
|
||||
return valid
|
||||
|
||||
|
||||
def test_block_mask_head_agnostic_and_masks_correctly():
|
||||
captured = {}
|
||||
|
||||
def fake_create_block_mask(mask_fn, *, B, H, Q_LEN, KV_LEN, device, _compile):
|
||||
captured.update(mask_fn=mask_fn, B=B, H=H, Q_LEN=Q_LEN, KV_LEN=KV_LEN)
|
||||
return "SENTINEL_BLOCK_MASK"
|
||||
|
||||
stub = SimpleNamespace(_block_masks=OrderedDict(), _device=torch.device("cpu"))
|
||||
layout = _SHORT_CLIP_LAYOUT
|
||||
with patch(f"{_DIT}.create_block_mask", side_effect=fake_create_block_mask) as cbm:
|
||||
first = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout)
|
||||
second = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout)
|
||||
|
||||
# Cached: the second call never rebuilds.
|
||||
assert first == "SENTINEL_BLOCK_MASK"
|
||||
assert second is first
|
||||
assert cbm.call_count == 1
|
||||
|
||||
# Head/batch-agnostic so SP head-sharding cannot change the mask.
|
||||
assert captured["B"] is None
|
||||
assert captured["H"] is None
|
||||
# Sized from the full clip (16 gen + 12 reference-video tokens), each side ceil'd to 128.
|
||||
assert captured["Q_LEN"] == 128
|
||||
assert captured["KV_LEN"] == 256
|
||||
|
||||
valid = _mask_probe(captured["mask_fn"])
|
||||
# Generation queries attend every generation token of the full clip, not the gen pad.
|
||||
assert valid(0, 0) and valid(5, 15)
|
||||
assert not valid(5, 16)
|
||||
# A pad query row attends nothing.
|
||||
assert not valid(16, 0)
|
||||
# Frame stride is the full clip's 4 tokens per frame, not this clip's 2: query 5 is in
|
||||
# generation frame 1 and attends reference-video frame 1 (keys 128..131) only. A mask
|
||||
# built from the clip's own grid would put query 5 in frame 2 and query 2 in frame 1.
|
||||
assert valid(5, 128) and valid(5, 131)
|
||||
assert not valid(5, 132)
|
||||
assert not valid(2, 128)
|
||||
# Frame 0 is the reference image; it has no reference-video counterpart.
|
||||
assert not valid(0, 128)
|
||||
# Frame 3 attends keys 136..139; 140 is past the 12 reference-video tokens (pad).
|
||||
assert valid(13, 136) and valid(13, 139)
|
||||
assert not valid(13, 140)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cache_kind", ["rope", "mask"])
|
||||
def test_geometry_cache_bounds_and_rebuilds(cache_kind):
|
||||
cache = OrderedDict()
|
||||
stub = SimpleNamespace(_device=torch.device("cpu"))
|
||||
if cache_kind == "rope":
|
||||
limit = 4
|
||||
stub._rope_cache = cache
|
||||
stub.rotary_emb = SimpleNamespace(
|
||||
forward_uncached=lambda positions: (positions.cos(), positions.sin())
|
||||
)
|
||||
|
||||
def lookup(size):
|
||||
return WanAnimate2Transformer3DModel._rope_tables(
|
||||
stub, (0, size, 1), (0, 2), (0, 2)
|
||||
)
|
||||
|
||||
context = nullcontext()
|
||||
else:
|
||||
limit = 2
|
||||
stub._block_masks = cache
|
||||
|
||||
def lookup(size):
|
||||
layout = _InContextLayout((size, 2, 2), (size - 1, 2, 2), (size, 2, 2))
|
||||
return WanAnimate2Transformer3DModel._get_block_mask_from_layout(
|
||||
stub, layout
|
||||
)
|
||||
|
||||
context = patch(
|
||||
f"{_DIT}.create_block_mask",
|
||||
side_effect=lambda fn, **kwargs: torch.tensor(
|
||||
[_mask_probe(fn)(0, i) for i in range(32)]
|
||||
),
|
||||
)
|
||||
with context:
|
||||
first = lookup(2)
|
||||
oldest_key = next(iter(cache))
|
||||
for size in range(3, limit + 2):
|
||||
lookup(size)
|
||||
assert lookup(2) is first
|
||||
lookup(limit + 2)
|
||||
assert oldest_key in cache
|
||||
for size in range(limit + 3, 2 * limit + 3):
|
||||
lookup(size)
|
||||
assert len(cache) == limit
|
||||
assert oldest_key not in cache
|
||||
rebuilt = lookup(2)
|
||||
torch.testing.assert_close(rebuilt, first, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_attention_pads_tokens_into_the_slots_the_block_mask_admits():
|
||||
# The mask is built once per request from the full clip while the attention pads every
|
||||
# clip's tokens; a frame stride or reference-video offset that drifts between the two
|
||||
# is a silent attention-pattern bug no shape check catches.
|
||||
layout = _SHORT_CLIP_LAYOUT
|
||||
captured = {}
|
||||
|
||||
def fake_flex_attention(q, k, v, *, block_mask, score_mod):
|
||||
captured.update(q=q, k=k, v=v, block_mask=block_mask)
|
||||
return torch.zeros_like(q)
|
||||
|
||||
batch, heads, dim = 1, 2, 8
|
||||
|
||||
def tokens(values, scale):
|
||||
base = torch.tensor(values, dtype=torch.float32).view(1, len(values), 1, 1)
|
||||
return (base * scale).expand(batch, len(values), heads, dim).clone()
|
||||
|
||||
# Token ids 1..4 are generation frame 0 (1, 2) and frame 1 (3, 4); 1, 2 x 1000 are the
|
||||
# reference-video frame.
|
||||
q, k, v = (
|
||||
tokens([1, 2, 3, 4], 1.0),
|
||||
tokens([1, 2, 3, 4], 10.0),
|
||||
tokens([1, 2, 3, 4], 100.0),
|
||||
)
|
||||
reference_k, reference_v = tokens([1, 2], 1000.0), tokens([1, 2], 10000.0)
|
||||
stub = SimpleNamespace(log_scale=0.0)
|
||||
with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
|
||||
out = WanAnimate2TransformerBlock._in_context_attention(
|
||||
stub, q, k, v, reference_k, reference_v, "SENTINEL_BLOCK_MASK", layout
|
||||
)
|
||||
|
||||
assert captured["block_mask"] == "SENTINEL_BLOCK_MASK"
|
||||
assert tuple(out.shape) == (batch, 4, heads, dim)
|
||||
q_padded = captured["q"].transpose(1, 2) # back to [B, L, N, D]
|
||||
k_padded = captured["k"].transpose(1, 2)
|
||||
v_padded = captured["v"].transpose(1, 2)
|
||||
assert tuple(q_padded.shape) == (
|
||||
batch,
|
||||
layout.padded_generation_video_len,
|
||||
heads,
|
||||
dim,
|
||||
)
|
||||
assert tuple(k_padded.shape) == (batch, layout.padded_kv_len, heads, dim)
|
||||
|
||||
def slot(t, i):
|
||||
return t[0, i, 0, 0].item()
|
||||
|
||||
# Generation tokens sit frame by frame at the full clip's 4-token stride.
|
||||
assert [slot(q_padded, i) for i in range(8)] == [1, 2, 0, 0, 3, 4, 0, 0]
|
||||
assert [slot(k_padded, i) for i in range(8)] == [10, 20, 0, 0, 30, 40, 0, 0]
|
||||
assert q_padded[:, 8:].abs().sum() == 0
|
||||
# Reference-video tokens start at padded_generation_video_len with the same stride.
|
||||
reference_start = layout.padded_generation_video_len
|
||||
assert [slot(k_padded, reference_start + i) for i in range(4)] == [1000, 2000, 0, 0]
|
||||
assert [slot(v_padded, reference_start + i) for i in range(4)] == [
|
||||
10000,
|
||||
20000,
|
||||
0,
|
||||
0,
|
||||
]
|
||||
assert k_padded[:, 8:reference_start].abs().sum() == 0
|
||||
assert k_padded[:, reference_start + 4 :].abs().sum() == 0
|
||||
|
||||
# Every slot that holds a real token is one the mask built from the same layout routes:
|
||||
# all gen pairs, and frame-1 queries (slots 4, 5) to the reference-video keys.
|
||||
valid = _mask_probe(layout.block_mask_fn())
|
||||
gen_slots = [0, 1, 4, 5]
|
||||
assert all(valid(qi, ki) for qi in gen_slots for ki in gen_slots)
|
||||
assert all(valid(qi, reference_start + ki) for qi in (4, 5) for ki in (0, 1))
|
||||
assert not any(valid(qi, reference_start + ki) for qi in (0, 1) for ki in (0, 1))
|
||||
|
||||
|
||||
def test_in_context_attention_hands_flex_a_score_mod_only_when_log_scale_is_nonzero():
|
||||
# At log_scale 0 the bias is an exact identity, so flex must run without a score_mod;
|
||||
# otherwise it must get the cached partial (compiled flex guards on its identity).
|
||||
layout = _SHORT_CLIP_LAYOUT
|
||||
captured = []
|
||||
|
||||
def fake_flex_attention(q, k, v, *, block_mask, score_mod):
|
||||
captured.append(score_mod)
|
||||
return torch.zeros_like(q)
|
||||
|
||||
gen, reference = torch.zeros(1, 4, 2, 8), torch.zeros(1, 2, 2, 8)
|
||||
with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
|
||||
for log_scale in (0.0, -1.3):
|
||||
WanAnimate2TransformerBlock._in_context_attention(
|
||||
SimpleNamespace(log_scale=log_scale),
|
||||
gen,
|
||||
gen,
|
||||
gen,
|
||||
reference,
|
||||
reference,
|
||||
"SENTINEL_BLOCK_MASK",
|
||||
layout,
|
||||
)
|
||||
assert captured[0] is None
|
||||
assert captured[1] is _make_score_mod(layout.max_tokens_per_frame, -1.3)
|
||||
|
||||
|
||||
def test_make_score_mod_is_cached_by_key():
|
||||
# Compiled flex_attention guards on score_mod identity; a fresh closure per call
|
||||
# would recompile every clip.
|
||||
assert _make_score_mod(4, 0.0) is _make_score_mod(4, 0.0)
|
||||
assert _make_score_mod(4, 0.0) is not _make_score_mod(4, 1.3)
|
||||
assert _make_score_mod(4, 0.0) is not _make_score_mod(8, 0.0)
|
||||
|
||||
|
||||
def test_score_mod_bias_region_matches_the_official_formula():
|
||||
# The official wan_animate_2_model.py adds log_scale to keys with index in [hw, 2*hw),
|
||||
# hw = full-clip tokens per frame (external literal: the distilled checkpoint depends on
|
||||
# it). In the padded [gen | reference-video] layout those are the keys of generation
|
||||
# frame 1; the reference-video slice at padded_generation_video_len is not biased.
|
||||
layout = _SHORT_CLIP_LAYOUT
|
||||
hw = layout.max_tokens_per_frame # 4
|
||||
score_mod = _make_score_mod(hw, 5.0)
|
||||
|
||||
def biased(kv_idx):
|
||||
out = score_mod(torch.tensor([1.0]), 0, 0, 0, torch.tensor(kv_idx))
|
||||
return out.item() == 6.0
|
||||
|
||||
probes = list(range(0, 3 * hw)) + [layout.padded_generation_video_len]
|
||||
assert [kv for kv in probes if biased(kv)] == [4, 5, 6, 7]
|
||||
|
||||
# Base weights ship log_scale 0.0: the score_mod must then be an exact identity.
|
||||
identity = _make_score_mod(hw, 0.0)
|
||||
score = torch.randn(5)
|
||||
for kv_idx in (0, hw, 2 * hw, layout.padded_generation_video_len):
|
||||
assert torch.equal(
|
||||
identity(score.clone(), 0, 0, 0, torch.tensor(kv_idx)), score
|
||||
)
|
||||
|
||||
|
||||
class _FakeAllToAll4D:
|
||||
"""Single-process all_to_all_4D for SP=2: split scatter_dim into (SP, .) and fold
|
||||
that factor onto gather_dim (and the inverse)."""
|
||||
|
||||
SP = 2
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, x, scatter_dim, gather_dim):
|
||||
self.calls.append((scatter_dim, gather_dim))
|
||||
b, s, h, c = x.shape
|
||||
sp = self.SP
|
||||
if (scatter_dim, gather_dim) == (2, 1):
|
||||
assert h % sp == 0
|
||||
return (
|
||||
x.reshape(b, s, sp, h // sp, c)
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.reshape(b, sp * s, h // sp, c)
|
||||
)
|
||||
if (scatter_dim, gather_dim) == (1, 2):
|
||||
assert s % sp == 0
|
||||
return (
|
||||
x.reshape(b, sp, s // sp, h, c)
|
||||
.permute(0, 2, 1, 3, 4)
|
||||
.reshape(b, s // sp, sp * h, c)
|
||||
)
|
||||
raise AssertionError((scatter_dim, gather_dim))
|
||||
|
||||
|
||||
def _reference_pass_stub(captured: list) -> SimpleNamespace:
|
||||
# Minimal block for forward_ref: identity norm/modulation, q/k/v straight from the stream,
|
||||
# attn1 records its keyword arguments.
|
||||
def attn1(q, k, v, **kwargs):
|
||||
captured.append({"q": q, "k": k, **kwargs})
|
||||
return q
|
||||
|
||||
return SimpleNamespace(
|
||||
local_num_heads=2,
|
||||
_modulate=lambda t: tuple(torch.zeros(1, 1, 8) for _ in range(6)),
|
||||
norm1=lambda x, shift, scale: x,
|
||||
_qkv=lambda x, n: (
|
||||
x.reshape(1, -1, n, 4),
|
||||
x.reshape(1, -1, n, 4) + 1,
|
||||
x.reshape(1, -1, n, 4) + 2,
|
||||
),
|
||||
attn1=attn1,
|
||||
_cross_and_ffn=lambda hidden, attn, *rest: hidden,
|
||||
)
|
||||
|
||||
|
||||
def _run_reference_pass(
|
||||
stub,
|
||||
*,
|
||||
seq_len: int,
|
||||
num_valid: int,
|
||||
sp_size: int,
|
||||
key_mask=None,
|
||||
k_cache=None,
|
||||
v_cache=None,
|
||||
cos=None,
|
||||
sin=None,
|
||||
):
|
||||
cos = torch.ones(num_valid, 2) if cos is None else cos
|
||||
sin = torch.zeros(num_valid, 2) if sin is None else sin
|
||||
with (
|
||||
patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size),
|
||||
patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)),
|
||||
patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x),
|
||||
):
|
||||
return WanAnimate2TransformerBlock.forward_ref(
|
||||
stub,
|
||||
hidden_states=torch.arange(seq_len * 8, dtype=torch.float32).reshape(
|
||||
1, seq_len, 8
|
||||
),
|
||||
encoder_hidden_states=torch.zeros(1, 4, 8),
|
||||
timestep_modulation=torch.zeros(1, 6, 8),
|
||||
reference_video_rope_cos=cos,
|
||||
reference_video_rope_sin=sin,
|
||||
reference_video_num_tokens=num_valid,
|
||||
index=0,
|
||||
k_cache={} if k_cache is None else k_cache,
|
||||
v_cache={} if v_cache is None else v_cache,
|
||||
reference_video_key_mask=key_mask,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_len,sp_size", [(6, 1), (8, 2)])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_reference_cache_reuses_rotated_key_and_preserves_padding(
|
||||
seq_len, sp_size, dtype
|
||||
):
|
||||
captured, keys, values = [], {}, {}
|
||||
angles = torch.arange(12, dtype=torch.float32).view(6, 2) / 7
|
||||
stub = _reference_pass_stub(captured)
|
||||
qkv = stub._qkv
|
||||
raw_keys = []
|
||||
|
||||
def project(x, n):
|
||||
projected = tuple(t.to(dtype) for t in qkv(x, n))
|
||||
raw_keys.append(projected[1])
|
||||
return projected
|
||||
|
||||
stub._qkv = project
|
||||
_run_reference_pass(
|
||||
stub,
|
||||
seq_len=seq_len,
|
||||
num_valid=6,
|
||||
sp_size=sp_size,
|
||||
k_cache=keys,
|
||||
v_cache=values,
|
||||
cos=angles.cos(),
|
||||
sin=angles.sin(),
|
||||
)
|
||||
raw = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 2, 4)
|
||||
expected = torch.cat(
|
||||
[
|
||||
_apply_rope_interleaved(
|
||||
(raw[:, :6] + 1).to(dtype), angles.cos(), angles.sin()
|
||||
),
|
||||
(raw[:, 6:] + 1).to(dtype),
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
torch.testing.assert_close(keys[0], expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(values[0], (raw + 2).to(dtype), rtol=0, atol=0)
|
||||
torch.testing.assert_close(keys[0], captured[0]["k"], rtol=0, atol=0)
|
||||
assert keys[0].data_ptr() == raw_keys[0].data_ptr()
|
||||
|
||||
|
||||
def test_reference_pass_passes_no_key_mask_when_the_sequence_has_no_padding():
|
||||
# A dense all-true key mask would route the reference stream through SDPA instead of the
|
||||
# configured attention backend; with no padding the pass must hand attn1 attn_mask=None and
|
||||
# RoPE the whole sequence.
|
||||
captured: list = []
|
||||
_run_reference_pass(
|
||||
_reference_pass_stub(captured), seq_len=6, num_valid=6, sp_size=1
|
||||
)
|
||||
assert len(captured) == 1
|
||||
assert captured[0]["attn_mask"] is None
|
||||
assert captured[0]["q"].shape == (1, 6, 2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_len,sp_size", [(4, 1), (6, 2)])
|
||||
def test_generation_reuses_reference_cache_without_rotating_it(seq_len, sp_size):
|
||||
stub = _reference_pass_stub([])
|
||||
captured = []
|
||||
|
||||
def attend(q, k, v, ref_k, ref_v, *rest):
|
||||
captured.append((q, k, ref_k, ref_v))
|
||||
return q
|
||||
|
||||
stub._in_context_attention = attend
|
||||
keys = {0: torch.randn(1, 2, 2, 4)}
|
||||
values = {0: torch.randn_like(keys[0])}
|
||||
saved_key = keys[0].clone()
|
||||
angles = torch.arange(8, dtype=torch.float32).view(4, 2) / 7
|
||||
hidden = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 8)
|
||||
with (
|
||||
patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size),
|
||||
patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)),
|
||||
patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x),
|
||||
):
|
||||
for _ in range(2):
|
||||
WanAnimate2TransformerBlock.forward_gen(
|
||||
stub,
|
||||
hidden_states=hidden,
|
||||
encoder_hidden_states=torch.zeros(1, 4, 8),
|
||||
timestep_modulation=torch.zeros(1, 6, 8),
|
||||
gen_cos=angles.cos(),
|
||||
gen_sin=angles.sin(),
|
||||
block_mask=None,
|
||||
layout=_SHORT_CLIP_LAYOUT,
|
||||
index=0,
|
||||
k_cache=keys,
|
||||
v_cache=values,
|
||||
)
|
||||
raw = hidden.view(1, seq_len, 2, 4)
|
||||
for q, k, ref_k, ref_v in captured:
|
||||
for actual, source in ((q, raw), (k, raw + 1)):
|
||||
expected = torch.cat(
|
||||
[
|
||||
_apply_rope_interleaved(source[:, :4], angles.cos(), angles.sin()),
|
||||
source[:, 4:],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
assert ref_k is keys[0] and ref_v is values[0]
|
||||
torch.testing.assert_close(keys[0], saved_key, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_reference_pass_masks_the_sp_tail_padding_with_the_per_clip_mask():
|
||||
# Gathered under SP the sequence carries tail padding: the pad keys are masked, and a mask
|
||||
# built once per clip by the caller is used as is.
|
||||
captured: list = []
|
||||
_run_reference_pass(
|
||||
_reference_pass_stub(captured), seq_len=8, num_valid=6, sp_size=2
|
||||
)
|
||||
mask = captured[0]["attn_mask"]
|
||||
assert mask is not None and mask.dtype == torch.bool and tuple(mask.shape) == (1, 8)
|
||||
assert mask[0, :6].all() and not mask[0, 6:].any()
|
||||
assert captured[0]["skip_sequence_parallel_override"] is True
|
||||
|
||||
per_clip = torch.tensor([[True] * 6 + [False] * 2])
|
||||
captured.clear()
|
||||
_run_reference_pass(
|
||||
_reference_pass_stub(captured),
|
||||
seq_len=8,
|
||||
num_valid=6,
|
||||
sp_size=2,
|
||||
key_mask=per_clip,
|
||||
)
|
||||
assert captured[0]["attn_mask"] is per_clip
|
||||
|
||||
|
||||
def test_gather_then_scatter_round_trips():
|
||||
# _sp_gather_qkv: [B,S_local,N_tp,C] -> [B,S_full,N_local,C]; _sp_scatter_out inverts it.
|
||||
fake = _FakeAllToAll4D()
|
||||
with (
|
||||
patch(
|
||||
f"{_BLOCK}._sp_gather_qkv",
|
||||
side_effect=lambda q, k, v: fake(
|
||||
torch.cat([q, k, v]), scatter_dim=2, gather_dim=1
|
||||
).chunk(3),
|
||||
) as gather,
|
||||
patch(
|
||||
f"{_BLOCK}._usp_output_all_to_all",
|
||||
side_effect=lambda x, head_dim: fake(x, scatter_dim=1, gather_dim=2),
|
||||
) as scatter,
|
||||
):
|
||||
q = torch.arange(1 * 4 * 8 * 16, dtype=torch.float32).reshape(1, 4, 8, 16)
|
||||
k = q + 1000.0
|
||||
v = q + 2000.0
|
||||
|
||||
gq, gk, gv = wan_animate_2_block._sp_gather_qkv(q, k, v)
|
||||
assert tuple(gq.shape) == (1, 8, 4, 16)
|
||||
assert tuple(gk.shape) == (1, 8, 4, 16)
|
||||
assert tuple(gv.shape) == (1, 8, 4, 16)
|
||||
|
||||
back = _sp_scatter_out(gq)
|
||||
assert tuple(back.shape) == tuple(q.shape)
|
||||
assert torch.equal(back, q)
|
||||
gather.assert_called_once_with(q, k, v)
|
||||
scatter.assert_called_once_with(gq, head_dim=2)
|
||||
|
||||
assert fake.calls == [(2, 1), (1, 2)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_clip_grid, expected",
|
||||
[
|
||||
# 4 frames x 4 tokens: 16 gen + 12 reference-video tokens, one 128 block each.
|
||||
((4, 2, 2), (4, 16, 12, 128, 128, 256)),
|
||||
# 4 frames x 100 tokens: 400 -> 512, 300 -> 384.
|
||||
((4, 10, 10), (100, 400, 300, 512, 384, 896)),
|
||||
],
|
||||
)
|
||||
def test_layout_pads_the_full_clip_geometry_to_128_token_blocks(
|
||||
full_clip_grid, expected
|
||||
):
|
||||
# The reference video has no counterpart for the generation stream's reference-image
|
||||
# frame, so its token count is one frame short of the generation stream's.
|
||||
layout = _InContextLayout((3, 2, 2), (2, 2, 2), full_clip_grid)
|
||||
assert (
|
||||
layout.max_tokens_per_frame,
|
||||
layout.max_generation_video_num_tokens,
|
||||
layout.max_reference_video_num_tokens,
|
||||
layout.padded_generation_video_len,
|
||||
layout.padded_reference_video_len,
|
||||
layout.padded_kv_len,
|
||||
) == expected
|
||||
|
||||
|
||||
class _KvTaggingTransformer:
|
||||
"""Fake DiT: ``build_reference_kv`` tags the K/V with the clip's reference-video latents,
|
||||
``__call__`` records the (clip_cond, reference_kv) pair each forward was handed."""
|
||||
|
||||
def __init__(self):
|
||||
self.context_calls = 0
|
||||
self.forward_calls: list[
|
||||
tuple[WanAnimate2ClipConditioning, WanAnimate2ReferenceKV]
|
||||
] = []
|
||||
|
||||
def build_reference_kv(self, *, clip_cond):
|
||||
latents = clip_cond.reference_video_latents
|
||||
return WanAnimate2ReferenceKV(k={0: latents}, v={0: latents})
|
||||
|
||||
def prepare_context(self, prompt_embeddings, image_embeddings):
|
||||
self.context_calls += 1
|
||||
return prompt_embeddings
|
||||
|
||||
def __call__(self, *, hidden_states, clip_cond, reference_kv, **_unused):
|
||||
self.forward_calls.append((clip_cond, reference_kv))
|
||||
return hidden_states.unsqueeze(0)
|
||||
|
||||
|
||||
class _TwoStepScheduler:
|
||||
timesteps = torch.tensor([1000.0, 500.0])
|
||||
order = 1
|
||||
|
||||
def set_timesteps(self, *, sigmas, device):
|
||||
pass
|
||||
|
||||
def step(self, noise_pred, t, sample, return_dict):
|
||||
return (sample,)
|
||||
|
||||
|
||||
def _clip_conditioning(reference_video_latents):
|
||||
return WanAnimate2ClipConditioning(
|
||||
reference_video_latents=reference_video_latents,
|
||||
generation_video_grid_sizes=(2, 1, 1),
|
||||
reference_video_frame_0_image_embeddings=torch.zeros(1, 1, 4),
|
||||
reference_video_condition=torch.zeros(20, 1, 2, 2),
|
||||
prompt_ref_embeddings=torch.zeros(1, 4),
|
||||
generation_condition=torch.zeros(20, 2, 2, 2),
|
||||
reference_video_grid_sizes=(1, 1, 1),
|
||||
full_clip_grid_sizes=(2, 1, 1),
|
||||
num_frames=4,
|
||||
init_noise=torch.zeros(16, 2, 2, 2),
|
||||
)
|
||||
|
||||
|
||||
def _batch_without_perf_recording(*, is_warmup=False) -> SimpleNamespace:
|
||||
"""The clip loop reads only the perf-recording fields off the batch (and hands it to
|
||||
set_forward_context); no metrics object means no step records."""
|
||||
return SimpleNamespace(
|
||||
metrics=None,
|
||||
perf_dump_path=None,
|
||||
is_warmup=is_warmup,
|
||||
rollout=False,
|
||||
return_trajectory_latents=False,
|
||||
)
|
||||
|
||||
|
||||
def _fake_denoising_stage(
|
||||
transformer, *, events: list[str]
|
||||
) -> WanAnimate2DenoisingStage:
|
||||
"""Stand-in for WanAnimate2DenoisingStage in _denoise_single_clip; the residency hooks
|
||||
record into ``events`` so a test can read the use interval back."""
|
||||
stage = WanAnimate2DenoisingStage.__new__(WanAnimate2DenoisingStage)
|
||||
stage.transformer = transformer
|
||||
stage.scheduler = _TwoStepScheduler()
|
||||
stage.step_profile = lambda: None
|
||||
stage._apply_nvtx_gate = lambda is_warmup: False
|
||||
stage.log_info = lambda *args: None
|
||||
stage.begin_declared_component_use = lambda *, component_name, module: (
|
||||
events.append(f"begin:{component_name}")
|
||||
)
|
||||
stage._finish_active_component_use = lambda: events.append("finish")
|
||||
return stage
|
||||
|
||||
|
||||
def _single_clip_request_state(guidance_scale: float) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
prompt_embeddings=torch.zeros(1, 4),
|
||||
negative_prompt_embeddings=torch.zeros(1, 4),
|
||||
reference_image_embeddings=torch.zeros(1, 1, 4),
|
||||
inputs=SimpleNamespace(num_inference_steps=2, guidance_scale=guidance_scale),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_warmup", [False, True])
|
||||
@pytest.mark.parametrize("guidance_scale", [1.0, 3.0])
|
||||
def test_each_clip_is_denoised_with_kv_from_its_own_reference_video(
|
||||
is_warmup, guidance_scale
|
||||
):
|
||||
"""Consecutive clips that share seed and clip index but not the reference video (two
|
||||
requests on one DiT) must each be denoised with K/V built from their own reference video,
|
||||
never with K/V left over from the previous one."""
|
||||
transformer = _KvTaggingTransformer()
|
||||
events = []
|
||||
stage = _fake_denoising_stage(transformer, events=events)
|
||||
stage.step_profile = lambda: events.append("profile")
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_config=SimpleNamespace(flow_shift=5.0),
|
||||
enable_cfg_parallel=False,
|
||||
disable_autocast=False,
|
||||
)
|
||||
request_state = _single_clip_request_state(guidance_scale=guidance_scale)
|
||||
first = _clip_conditioning(torch.full((16, 1, 2, 2), 1.0))
|
||||
second = _clip_conditioning(torch.full((16, 1, 2, 2), 2.0))
|
||||
|
||||
with patch(
|
||||
f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu")
|
||||
):
|
||||
for clip_cond in (first, second):
|
||||
WanAnimate2DenoisingStage._denoise_single_clip(
|
||||
stage,
|
||||
batch=_batch_without_perf_recording(is_warmup=is_warmup),
|
||||
server_args=server_args,
|
||||
request_state=request_state,
|
||||
clip_condition=clip_cond,
|
||||
)
|
||||
|
||||
# 2 clips x 2 steps; every forward of a clip gets the K/V built from that clip's latents.
|
||||
branches = 2 if guidance_scale > 1 else 1
|
||||
assert len(transformer.forward_calls) == 4 * branches
|
||||
assert transformer.context_calls == 2 * branches
|
||||
assert events.count("profile") == (0 if is_warmup else 4)
|
||||
for clip_cond, reference_kv in transformer.forward_calls:
|
||||
assert reference_kv.k[0] is clip_cond.reference_video_latents
|
||||
assert (
|
||||
transformer.forward_calls[0][1]
|
||||
is not transformer.forward_calls[2 * branches][1]
|
||||
)
|
||||
|
||||
|
||||
def test_dit_work_of_a_clip_runs_inside_one_declared_transformer_use():
|
||||
"""The stage declares three component uses, so the residency manager does not activate
|
||||
the DiT at stage entry; the reference pass and every CFG forward of a clip must run after
|
||||
the transformer use begins and before it is finished, or an offloaded DiT is never
|
||||
brought back to the device."""
|
||||
events: list[str] = []
|
||||
|
||||
class _EventTransformer(_KvTaggingTransformer):
|
||||
def build_reference_kv(self, *, clip_cond):
|
||||
events.append("reference_kv")
|
||||
return super().build_reference_kv(clip_cond=clip_cond)
|
||||
|
||||
def __call__(self, **kwargs):
|
||||
events.append("forward")
|
||||
return super().__call__(**kwargs)
|
||||
|
||||
def prepare_context(self, *args):
|
||||
events.append("context")
|
||||
return super().prepare_context(*args)
|
||||
|
||||
stage = _fake_denoising_stage(_EventTransformer(), events=events)
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_config=SimpleNamespace(flow_shift=5.0),
|
||||
enable_cfg_parallel=False,
|
||||
disable_autocast=False,
|
||||
)
|
||||
with patch(
|
||||
f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu")
|
||||
):
|
||||
WanAnimate2DenoisingStage._denoise_single_clip(
|
||||
stage,
|
||||
batch=_batch_without_perf_recording(),
|
||||
server_args=server_args,
|
||||
# guidance_scale 3.0: two forwards (cond, uncond) per step.
|
||||
request_state=_single_clip_request_state(guidance_scale=3.0),
|
||||
clip_condition=_clip_conditioning(torch.zeros(16, 1, 2, 2)),
|
||||
)
|
||||
|
||||
assert events == [
|
||||
"begin:transformer",
|
||||
"reference_kv",
|
||||
"context",
|
||||
"forward",
|
||||
"context",
|
||||
"forward",
|
||||
"forward",
|
||||
"forward",
|
||||
"finish",
|
||||
]
|
||||
|
||||
|
||||
def _ensure_single_process_parallel_runtime() -> None:
|
||||
# The DiT's parallel linear layers need the TP group even on a meta device.
|
||||
if model_parallel_is_initialized():
|
||||
return
|
||||
ensure_distributed_env_defaults()
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
def test_dit_constructs_only_its_own_blocks():
|
||||
"""Every block the DiT constructs is the in-context block; the parent's block set is not
|
||||
built and then replaced (40 block objects per construction, allocated on whatever device
|
||||
the loader initializes on)."""
|
||||
_ensure_single_process_parallel_runtime()
|
||||
config = WanAnimate2Config()
|
||||
config.arch_config.num_layers = 2
|
||||
constructed: list[type] = []
|
||||
original_init = WanTransformerBlock.__init__
|
||||
|
||||
def recording_init(self, *args, **kwargs):
|
||||
constructed.append(type(self))
|
||||
original_init(self, *args, **kwargs)
|
||||
|
||||
with (
|
||||
patch.object(WanTransformerBlock, "__init__", recording_init),
|
||||
torch.device("meta"),
|
||||
):
|
||||
model = WanAnimate2Transformer3DModel(config, hf_config={})
|
||||
|
||||
assert [type(block) for block in model.blocks] == [WanAnimate2TransformerBlock] * 2
|
||||
assert constructed == [WanAnimate2TransformerBlock] * 2
|
||||
|
||||
|
||||
def test_dit_inherits_shared_wan_projections_and_output_head():
|
||||
assert WanAnimate2TransformerBlock._qkv is WanTransformerBlock._qkv
|
||||
assert (
|
||||
WanAnimate2TransformerBlock._cross_and_ffn is WanTransformerBlock._cross_and_ffn
|
||||
)
|
||||
assert (
|
||||
WanAnimate2Transformer3DModel._unpatchify is WanTransformer3DModel._unpatchify
|
||||
)
|
||||
@@ -204,6 +204,39 @@ class TestFitAutoResidencyProbe:
|
||||
assert fitted.width * fitted.height <= 832 * 480
|
||||
|
||||
|
||||
class TestProbeFitTerminates:
|
||||
def test_pinned_frame_count_ends_the_shrink_ladder(self):
|
||||
"""A model whose sampling params pin num_frames must not hang the probe fit.
|
||||
|
||||
The lightened request comes back at the same size, so the fit must stop
|
||||
instead of retrying the same shape forever. The request is larger than the
|
||||
bounded warmup, so the measured-shape floor does not end the ladder first.
|
||||
"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.gpu_worker import (
|
||||
fit_auto_residency_probe,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class _PinnedFrames(SamplingParams):
|
||||
def __post_init__(self) -> None:
|
||||
self.num_frames = 37
|
||||
super().__post_init__()
|
||||
|
||||
req = Req(sampling_params=_PinnedFrames(width=576, height=1024))
|
||||
fitted, estimate, steps = fit_auto_residency_probe(
|
||||
req,
|
||||
records=[_record(640, 800, 37, peak_gib=54.0)],
|
||||
free_bytes=20 << 30,
|
||||
total_bytes=80 << 30,
|
||||
server_args=_server_args(),
|
||||
)
|
||||
assert steps == 0
|
||||
assert (fitted.width, fitted.height, fitted.num_frames) == (576, 1024, 37)
|
||||
assert estimate is not None and estimate > 20 << 30
|
||||
|
||||
|
||||
class TestOutOfMemoryClassification:
|
||||
def test_allocation_failures_from_libraries_count_as_out_of_memory(self):
|
||||
from sglang.multimodal_gen.runtime.server_warmup import _is_out_of_memory
|
||||
|
||||
Reference in New Issue
Block a user