From f1e0472fcc7dc34e2b9daaef1b41c9ffe53f487c Mon Sep 17 00:00:00 2001 From: Ankit Dhall Date: Fri, 9 Oct 2026 14:46:23 +0200 Subject: [PATCH] [diffusion] model: support Wan-Animate-2 (#42941) Co-authored-by: Ankit Dhall <8938083+ankitdhall@users.noreply.github.com> Co-authored-by: Mick Qian --- docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx | 377 ++++ docs/docs.json | 3 +- docs/docs/sglang-diffusion/api/openai_api.mdx | 15 + docs/src/snippets/diffusion/model-catalog.jsx | 5 + .../configs/models/dits/__init__.py | 2 + .../configs/models/dits/wan_animate_2.py | 130 ++ .../configs/pipeline_configs/wan.py | 147 +- .../multimodal_gen/configs/sample/wan.py | 161 ++ .../component_loaders/component_loader.py | 5 +- .../runtime/managers/gpu_worker.py | 18 +- .../runtime/models/dits/wan_animate_2.py | 445 +++++ .../models/dits/wan_animate_2_block.py | 532 +++++ .../dits/wan_animate_2_clip_conditioning.py | 67 + .../runtime/models/dits/wanvideo.py | 196 +- .../scheduling_dpm_solver_multistep.py | 38 +- .../pipelines/wan_animate_2_pipeline.py | 102 + .../runtime/pipelines_core/stages/decoding.py | 31 +- .../pipelines_core/stages/denoising.py | 51 +- .../wan_animate_2/__init__.py | 22 + .../wan_animate_2/audio.py | 68 + .../wan_animate_2/before_denoising.py | 706 +++++++ .../wan_animate_2/denoising.py | 532 +++++ .../wan_animate_2/encoder_adapters.py | 210 ++ .../wan_animate_2/preprocess.py | 324 +++ .../server/consistency_thresholds/h100.json | 12 + .../multimodal_gen/test/server/gpu_cases.py | 45 + .../test/server/perf_baselines/h100.json | 54 + .../test/server/testcase_configs.py | 33 + .../sglang/multimodal_gen/test/test_utils.py | 5 +- .../test/unit/test_wan_animate_2_config.py | 1763 +++++++++++++++++ .../unit/test_wan_animate_2_parallelism.py | 799 ++++++++ .../unit/test_warmup_probe_degradation.py | 33 + 32 files changed, 6815 insertions(+), 116 deletions(-) create mode 100644 docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx create mode 100644 python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py diff --git a/docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx b/docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx new file mode 100644 index 0000000000..54f6c02582 --- /dev/null +++ b/docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx @@ -0,0 +1,377 @@ +--- +title: Wan-Animate-2 +description: Run Wan-Animate-2 character animation (reference image + reference video) with SGLang Diffusion. +metatags: + description: "Deploy and use the Wan2.2-Animate-2 14B character-animation model with SGLang Diffusion: reference-image + reference-video conditioning, single-expert native pipeline, config-file and /v1/videos invocation, and multi-GPU presets." +--- + +import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx'; + + + +## 1. Model Introduction + +[Wan-Animate-2](https://huggingface.co/Wan-AI/Wan2.2-Animate-2-14B-Diffusers) is a +14B character-animation model in the Wan2.2 family. Instead of the text- or +image-only conditioning of the other Wan2.2 checkpoints, it animates a **reference +image** so that it follows the motion of a **reference video**, producing a new clip +of that subject performing the reference video's motion. + +Use **`Wan-AI/Wan2.2-Animate-2-14B-Diffusers`** as `--model-path`. The distilled +checkpoint (`Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers`, `log_scale=-1.3`, +10 steps, no CFG) is not supported in this release. + +Unlike the A14B T2V/I2V checkpoints, Wan-Animate-2 is a **single-expert** model: +it has one `guidance_scale` (default `3.0`), no low-noise expert +(`guidance_scale_2`) and no boundary switching. Diffusers >= 0.40.0 serves +Wan-Animate-2 as a modular pipeline (`WanAnimate2ModularPipeline`); SGLang +implements the model natively and loads the same +`Wan-AI/Wan2.2-Animate-2-14B-Diffusers` checkpoint. There is no diffusers backend +fallback. + + +Wan-Animate-2 is served through the native `WanAnimate2Pipeline`, selected +automatically from the checkpoint's `model_index.json`. It takes **two** inputs, a +reference image and a reference video, so it is invoked differently from the +other Wan2.2 models. See [section 4](#4-model-invocation). + + +### 1.1 Inputs + +| Input | Field | Passed via | +| --- | --- | --- | +| Reference image (the subject to animate) | `image_path` / `input_reference` | `--config` YAML for `sglang generate`; the `input_reference` part of a `POST /v1/videos` request | +| Reference video (the motion to follow) | `video_path` | `--video-path` or `--config` YAML for `sglang generate`; the `video_path` field in the `/v1/videos` request body or `extra_body` | + +Both inputs are the base `image_path` / `video_path` sampling fields shared with the +other video models; Wan-Animate-2 adds no model-specific input field. Offline runs +pass both through a `--config` file (or `--image-path` / `--video-path`), and the +online API forwards `video_path` into the pipeline while the reference image rides +`input_reference`. Local paths must be readable by the server process. HTTP(S) +video URLs use SGLang's shared media loader before decoding, without a disk cache. + +### 1.2 Defaults + +| Parameter | Default | Notes | +| --- | --- | --- | +| Resolution (`width`×`height`, the API's `size`) | `640`×`800` | A pixel-area budget, not the output dimensions: the frame keeps the reference image's aspect ratio and is the largest 16-aligned canvas with at most `width × height` pixels (see **Output resolution** below). `720`×`1280` is the larger supported budget. | +| `clip_len` | `37` | Per-clip denoising chunk length; must be `4k+1` (VAE temporal stride 4), e.g. `37`, `65`, `81`. Other values are rounded to `4k+1` with the same `k` and the rounding is logged (`40` becomes `41`; `38` and `39` become `37`); values below `5` are rejected. | +| `fps` | `16` | The reference video is resampled to this rate and the output MP4 is written at it; duration is preserved. | +| `num_inference_steps` | `40` | Fewer steps (e.g. `20`) run faster. | +| `guidance_scale` | `3.0` | Single expert; there is no `guidance_scale_2`. | +| `prompt_ref` | `人物动作的参考视频` ("reference video of the character's motion") | Text for the reference-video branch. The official demo conditions that branch on this fixed description of the reference video rather than on the scene prompt; leave it unchanged unless reproducing a different reference setup. | +| `enable_audio` | `true` | Keep the reference video's audio track on the output (see **Audio** below). Per request, like `clip_len`. | + +**Output length.** The output has as many frames as the (resampled) reference +video, not `clip_len`. The video is generated clip by clip with `clip_len` frames +per chunk; a larger `clip_len` improves temporal coherence at the cost of VRAM. +`clip_len=81` roughly doubles the attention tokens of the default and OOMs a +single 80 GB GPU. `num_frames` (and the API's `seconds`, which maps to it) does +not control the length: it is ignored and the server logs that it was, so leave +it unset. + +**Output resolution.** `width`×`height` (the API's `size`) is a pixel-area budget, +not the output dimensions. The reference image's aspect ratio is kept, the frame is +the largest canvas with at most `width × height` pixels whose sides are multiples of +16, and the reference image and the driving-video frames are letterboxed onto it; +the output MP4 has the canvas size. At the default `640`×`800` (512,000 pixels) a +9:16 reference gives `528`×`944`, a 1:1 reference gives `704`×`704`, and only a 4:5 +reference gives exactly `640`×`800`. To obtain a specific output size, request a +budget with the same aspect ratio as the reference image (for a 9:16 image, +`576`×`1024` maps onto itself). This differs from the OpenAI Videos API and from the +other SGLang diffusion pipelines, where `size` is the literal output resolution. + +**Audio.** The output MP4 carries the reference video's audio track, as the +official pipeline does. Set `enable_audio=false` on a request (a `--config` field +offline, a form field or `extra_body` entry online) for a silent output. If the +track cannot be decoded, the output is silent and the server logs one warning per +process; a reference video without an audio track also gives a silent output. + +**Lossless conditioning reuse.** Reference-video keys are rotated once and cached per clip, then reused by all +denoising steps and both CFG branches. This removes repeated RoPE computation +without changing the attention mask, precision, or sampling schedule. The cache +uses the existing key storage and is released after the clip; it is not an +approximate timestep-skipping cache. +Clips containing only padding that would be removed from the final video are not +denoised. The padding and conditioning of every retained clip are unchanged. + +Text encoding uses the native `TextEncodingStage` and its conditioning cache and +residency management. CFG-parallel dispatch shares the native CFG utilities while +retaining the single-GPU arithmetic order. +Each clip uses the shared denoising loop for step profiling, NVTX ranges, progress, +and DiT release. Reference K/V preparation and clip-specific prediction remain +model hooks; clip decoding still feeds the next clip's conditioning. This does +not enable cache-dit or breakable CUDA graphs for this model. +Clip decoding uses the shared `DecodingStage.decode_raw` path for VAE compilation, +tiling and OOM diagnostics. Raw decoded pixels are kept for the next clip: no +extra conversion to and from `[0, 1]` is introduced. The adapter preserves the +existing FP32 VAE execution and per-channel latent scaling. +QKV projections, cross-attention, FFN and the output head are inherited from the +native Wan implementation. Ulysses exchanges use the shared packed-QKV path; +the model-specific reference attention mask and K/V lifecycle are retained. +Text and image conditioning projections are reused within each clip and CFG +branch, rather than recomputed at every timestep. These projected tensors are +released after the clip and never reused across requests. Decoded output is +copied to CPU and clip-local intermediates are released; only the decoded +overlap frames needed by the next clip remain on the GPU. +Geometry caches retain at most four RoPE tables and two attention masks, rather +than accumulating GPU tensors for every request size. Evicted entries are +recomputed without changing the results; frequently alternating among more +geometries can add setup work. + +Measured on one H200, with a 576×1024 budget, 8 steps, CFG 3, seed 42, +`clip_len=37`, text-encoder CPU offload, and MP4 plus raw-frame output, the initial +reuse changes reduced warm request latency from 42.30s to 41.34s (two requests +per variant). With `clip_len=17`, omitting a fully cropped trailing clip reduced +59.60s to 40.45s (one warm request per variant). Raw RGB frames were bit-identical +and peak allocated VRAM did not increase. These are workload-specific results, +not general speedup claims; both comparisons used PyTorch 2.13.0+cu130, +Transformers 5.12.1, and Diffusers 0.37.0. The baseline was the initial native +implementation, not the official Diffusers implementation. + +## 2. SGLang-diffusion Installation + +SGLang-diffusion offers multiple installation methods. You can choose the most +suitable installation method based on your hardware platform and requirements. + +Please refer to the [official SGLang-diffusion installation guide](/docs/sglang-diffusion/installation) +for installation instructions. + +## 3. Model Deployment + +### 3.1 Basic Configuration + +On a single high-VRAM GPU, launch the native pipeline with: + +```bash Command +sglang serve \ + --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \ + --port 30010 +``` + +Routing to the native Wan-Animate-2 pipeline is automatic from the checkpoint's +`model_index.json` (`_class_name: WanAnimate2Pipeline`); pass +`--pipeline-class-name WanAnimate2Pipeline` only for a local mirror that lacks +`model_index.json`. + +### 3.2 Configuration Tips + +See [Performance Optimization](/docs/sglang-diffusion/performance-optimization) for +acceleration features and their runtime requirements. + +- `--num-gpus {NUM_GPUS}`: Number of GPUs to use. +- `--tp-size {TP_SIZE}`: Tensor parallelism size. +- `--ulysses-degree {ULYSSES_DEGREE}`: DeepSpeed-Ulysses-style sequence parallelism (USP). The **ring degree must be 1** for this model; `--ring-degree` greater than 1 is rejected with `NotImplementedError`. Combines with `--tp-size`: Ulysses splits the TP-local attention heads, so `--tp-size` must divide the 40 heads and `--ulysses-degree` must divide the per-shard count (see [section 3.3](#3-3-multi-gpu-and-memory-presets)). +- `--enable-cfg-parallel`: Split the guided and unguided branches across GPUs. +- `--text-encoder-cpu-offload`: CPU-offload the text encoder to save memory; it is moved to the GPU for the one encode per request. + +### 3.3 Multi-GPU and memory presets + +| Target | Recommended server flags | Notes | +| --- | --- | --- | +| 1 GPU, 96 GB or more | *(no extra flags)* | DiT resident: about 70 GiB peak per GPU at the default workload. Add `--text-encoder-cpu-offload` to keep the fp32 text encoder off the GPU. | +| 1 GPU, 80 GB | `--dit-layerwise-offload --layerwise-offload-components transformer` | Streams the DiT layer by layer: about 40 GiB peak at the same end-to-end time as the resident DiT; output identical. | +| 2+ GPUs, long sequences | `--num-gpus N --ulysses-degree N` | Sequence parallel across ranks (ring degree stays 1). | +| 2+ GPUs, large DiT | `--num-gpus N --tp-size N` | Tensor parallel across attention heads. With `--tp-size` smaller than `--num-gpus` (or than half of it with `--enable-cfg-parallel`), the leftover GPUs are auto-assigned to Ulysses sequence parallelism (last row). | +| 2 GPUs, split CFG | `--num-gpus 2 --enable-cfg-parallel` | Splits the guided/unguided branches. | +| 2 GPUs, 80 GB cards | `--num-gpus 2 --enable-cfg-parallel --dit-layerwise-offload --layerwise-offload-components transformer --encoder-parallel replicate --vae-config.use-parallel-decode false` with `SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D=1` | Same speed as resident CFG-parallel at about 43 GiB per GPU, output bit-identical to a single GPU (the CI configuration). | +| 4 GPUs | `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2` | CFG-parallel across two pairs of Ulysses ranks: the fastest 4-GPU layout; pass the flags explicitly to pin the layout instead of relying on the automatic assignment. Ulysses shards the attention sequence, so 4-GPU outputs match the single-GPU video to about 30 dB mean PSNR rather than bit for bit. On 80 GB cards add `--dit-layerwise-offload --layerwise-offload-components transformer` (about 35 GiB per GPU, same speed). | +| FSDP sharding | `--use-fsdp-inference` is **not supported** | Rejected at startup, before any weights load; use `--tp-size` to shard the DiT or `--dit-layerwise-offload` to cut peak memory. | +| 4+ GPUs, TP x Ulysses | `--num-gpus 4 --tp-size 2 --ulysses-degree 2` | Tensor parallel across attention heads, Ulysses across the TP-local heads: `--tp-size` must divide the 40 heads and `--ulysses-degree` the per-shard count (`40 / tp`). The memory-leaning 4-GPU layout: about 33 GiB peak per GPU (25 GiB torch reserved), the lowest 4-GPU peak after `--tp-size 4`, at about 13 percent slower than CFG-parallel x tensor parallel and 27 percent slower than the default CFG-parallel x Ulysses layout; use the default for speed and this layout for memory. CFG-parallel can be added on top with twice the GPUs (`--num-gpus 8 --enable-cfg-parallel --tp-size 2 --ulysses-degree 2`). | + +## 4. Model Invocation + +### 4.1 Generate offline with a config file + +The offline path passes both inputs (and the sampling parameters) through a +`--config` YAML: + +```yaml wan_animate_2_run.yaml +prompt: "a person dancing" +image_path: /path/to/reference.png +video_path: /path/to/reference_video.mp4 +clip_len: 37 +height: 800 +width: 640 +num_inference_steps: 40 +guidance_scale: 3.0 +fps: 16 +seed: 42 +``` + +```bash Command +sglang generate \ + --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \ + --config wan_animate_2_run.yaml \ + --num-gpus 1 --text-encoder-cpu-offload --pin-cpu-memory \ + --save-output --output-path outputs --output-file-name wan_animate_2.mp4 +``` + +Any value in the YAML can also be passed as a CLI flag (for example +`--image-path`, `--video-path`, `--num-inference-steps`, `--guidance-scale`, +`--seed`). + +### 4.2 Serve and request over HTTP + +`POST /v1/videos` creates an asynchronous job. Send the **reference image** as the +`input_reference` part and the **reference video** as `video_path`, then poll +the job and download the finished MP4. `size` is the pixel-area budget described in +[section 1.2](#1-2-defaults); the output dimensions follow the reference image's +aspect ratio. + +```bash Command +job_id=$(curl -sS -X POST http://127.0.0.1:30010/v1/videos \ + --form-string "prompt=a person dancing" \ + --form-string "video_path=/path/to/reference_video.mp4" \ + --form-string "clip_len=37" \ + --form-string "size=640x800" \ + --form-string "num_inference_steps=40" \ + --form-string "guidance_scale=3.0" \ + --form-string "fps=16" \ + --form-string "seed=42" \ + --form "input_reference=@/path/to/reference.png;type=image/png" \ + | python -c 'import json, sys; print(json.load(sys.stdin)["id"])') + +while true; do + status=$(curl -sS "http://127.0.0.1:30010/v1/videos/${job_id}" \ + | python -c 'import json, sys; print(json.load(sys.stdin)["status"])') + [ "$status" = "completed" ] && break + [ "$status" = "failed" ] && exit 1 + sleep 1 +done + +curl -sS -L "http://127.0.0.1:30010/v1/videos/${job_id}/content" \ + -o wan_animate_2.mp4 +``` + +The same request from the OpenAI Python client. The reference image is the +`input_reference` file, and `video_path` (a path the server can read) and +`clip_len` ride `extra_body`: + +```python Python +from openai import OpenAI + +client = OpenAI(api_key="EMPTY", base_url="http://127.0.0.1:30010/v1") +video = client.videos.create( + model="Wan-AI/Wan2.2-Animate-2-14B-Diffusers", + prompt="a person dancing", + input_reference=open("/path/to/reference.png", "rb"), + extra_body={ + "video_path": "/path/to/reference_video.mp4", + "clip_len": 37, + "guidance_scale": 3.0, + "enable_audio": False, # silent output; the reference video's audio is kept by default + }, +) +``` + +For more API usage and request examples, see the +[SGLang Diffusion OpenAI API](/docs/sglang-diffusion/api/openai_api) reference. + +### 4.3 Reducing memory + +When a single GPU is tight on VRAM, stream the DiT one layer at a time instead of +resident-loading it: + +```bash Command +sglang serve \ + --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \ + --dit-layerwise-offload \ + --layerwise-offload-components transformer \ + --port 30010 +``` + +Combine with `--text-encoder-cpu-offload --pin-cpu-memory` to also keep the text +encoder off the GPU between uses. Lowering `clip_len` also reduces peak memory. + +## 5. Benchmark + +Test Environment: + +- Hardware: NVIDIA GB200 GPU (1x, 2x and 4x; 189 GB per GPU) +- Model: Wan-AI/Wan2.2-Animate-2-14B-Diffusers +- sglang diffusion version: 0.5.22 +- Workload: `size` 640x800 (pixel-area budget) with a 16:9 reference image, so the frame is 960x528; a 237-frame reference video at `fps` 30 (7 clips of `clip_len` 37); 40 steps, CFG 3.0, seed 123; `--text-encoder-cpu-offload --warmup-mode request`. E2E is the request time after the warmup request, so it excludes the one-off flex-attention compile. + +`bench_serving` cannot pass `video_path`, so the benchmark runs offline +with `sglang generate --config` (the YAML from [section 4.1](#4-1-generate-offline-with-a-config-file)) +and reads timings from `--perf-dump-path`. `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1` +synchronizes the device so the per-stage timings are accurate. + + + + **Benchmark Command** (1 GPU shown; add the flags from the table for the other rows): + ```shell Command + SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 sglang generate \ + --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \ + --config wan_animate_2_run.yaml \ + --num-gpus 1 --text-encoder-cpu-offload --warmup-mode request \ + --save-output --perf-dump-path wan_animate_2_perf.json + ``` + + **Result** (peak memory per GPU from `nvidia-smi`): + + | Config | GPUs | Extra flags | E2E (s) | Peak per GPU (GiB) | + | --- | ---: | --- | ---: | ---: | + | 1 GPU, DiT resident | 1 | none | 993 | 69.5 | + | 1 GPU, layerwise DiT offload | 1 | `--dit-layerwise-offload --layerwise-offload-components transformer` | 996 | 40.2 | + | 2 GPUs, CFG-parallel | 2 | `--enable-cfg-parallel` | 513 | 67.3 | + | 2 GPUs, CFG-parallel, replicated encoders, serial VAE decode | 2 | `--enable-cfg-parallel --encoder-parallel replicate --vae-config.use-parallel-decode false` | 515 | 71.0 | + | 2 GPUs, CFG-parallel, layerwise DiT offload, bit-identical to 1 GPU | 2 | `--enable-cfg-parallel --dit-layerwise-offload --layerwise-offload-components transformer --encoder-parallel replicate --vae-config.use-parallel-decode false` with `SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D=1` | 512 | 43.0 | + | 2 GPUs, Ulysses | 2 | `--ulysses-degree 2` | 547 | 58.1 | + | 2 GPUs, tensor parallel | 2 | `--tp-size 2` | 613 | 36.6 | + | 4 GPUs, framework default (CFG2 x SP2, KV-gather exchange) | 4 | none | 283 | 58.4 | + | 4 GPUs, CFG-parallel x Ulysses | 4 | `--enable-cfg-parallel --ulysses-degree 2` | 283 | 58.4 | + | 4 GPUs, CFG-parallel x Ulysses, layerwise DiT offload | 4 | `--enable-cfg-parallel --ulysses-degree 2 --dit-layerwise-offload --layerwise-offload-components transformer --encoder-parallel replicate --vae-config.use-parallel-decode false` | 291 | 35.2 | + | 4 GPUs, Ulysses | 4 | `--ulysses-degree 4` | 317 | 49.0 | + | 4 GPUs, CFG-parallel x tensor parallel | 4 | `--enable-cfg-parallel --tp-size 2` | 318 | 37.5 | + | 4 GPUs, tensor parallel x Ulysses | 4 | `--tp-size 2 --ulysses-degree 2` | 361 | 33.5 | + | 4 GPUs, tensor parallel | 4 | `--tp-size 4` | 379 | 27.3 | + + Layerwise DiT offload cuts the 1-GPU peak by 29 GiB at the same end-to-end time. + The fastest 4-GPU layout is 3.5x faster than one GPU. The framework default and the explicit CFG-parallel x Ulysses + layout run at the same speed and produce the same output; the explicit flags pin the layout. + Tensor parallel x Ulysses is the memory-leaning 4-GPU layout: the lowest 4-GPU peak after `--tp-size 4`, about 13 percent slower than CFG-parallel x tensor parallel and 27 percent slower than the default. + Adding `--encoder-parallel replicate --vae-config.use-parallel-decode false` to the CFG-parallel x Ulysses row costs about 1 percent (287 s, 62.5 GiB) and does not make the output bit-identical to a single GPU: every Ulysses layout matches the single-GPU video to about 30 dB mean PSNR. + The layerwise rows are the configurations for 80 GB GPUs. With the replicated encoders, serial VAE decode and the + channels-last VAE layout (`SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D=1`) the 2-GPU CFG-parallel output is bit-identical to a single GPU. + + + +### 5.1 Native regression coverage + +A separate regression used two B300 GPUs, CFG-parallel with replicated encoders +and serial VAE decode, a 576x1024 area budget, 40 steps, `clip_len=37`, CFG=3, +and seed 42. The driving input was looped to 64 frames to exercise two clips; +this is a bounded regression workload, not a long-video quality benchmark. +Both revisions (`4ee69b4cf9ea` and `04725c1c2900`) used PyTorch 2.14.1+cu130, +Transformers 5.19.0, Diffusers 0.37.0, and FlashAttention 4.0.0b34 on the same node. + +Cold and warm raw RGB outputs were byte-identical to the pre-optimization native +implementation, both with resident DiT weights and with layerwise DiT offload. +The single warm samples measured 112.29 s before and 112.40 s after optimization; +offload measured 113.46 s. This does not establish a significant latency change. +Rank 0's request-stage peak allocated memory was 52.24, 52.23, and 23.93 GiB, +respectively; these are not loading peaks or whole-device physical memory peaks. + +An H200 HTTP smoke test also covered image upload, asynchronous job polling and +download, request warmup, repeated requests, URL video input, and reference-audio +passthrough. Repeated decoded RGB outputs were identical. + +These checks preserve the existing native output, not bitwise parity with the +newer Diffusers modular pipeline. The native sampler follows the original Wan +sigma grid starting at exactly 1.0 and retains its BF16 conditioning conversion; +the modular pipeline's sampling and preprocessing differ. Official-reference +accuracy must be validated separately with pinned versions and aligned settings. + +## 6. Troubleshooting + +- **OOM**: lower `clip_len` (for example `37` instead of `65`/`81`) or add `--dit-layerwise-offload --layerwise-offload-components transformer`. +- **`--ring-degree` greater than 1** is rejected at startup with `NotImplementedError`. Use `--ulysses-degree` for sequence parallelism and keep the ring degree at 1. +- **`--ulysses-degree` that does not divide the TP-local attention heads** (40 / `--tp-size`; with TP 1, degrees 3, 6, 16, ... are rejected) fails at startup with `ValueError`; with TP 1 use 2, 4, 5, 8, 10, 20 or 40. The same check covers leftover GPUs auto-assigned to sequence parallelism (for example `--num-gpus 8 --tp-size 2` runs TP 2 x Ulysses 4). +- **FSDP** (`--use-fsdp-inference`) is not supported for this model and is rejected at startup; use `--tp-size` or DiT layerwise offload for memory. +- **`--attention-backend`** (or `--component-attention-backends transformer=...`) is rejected at startup with `ValueError`: the in-context self-attention runs on torch `flex_attention` only. The text encoder still accepts `--component-attention-backends text_encoder=`. +- **`enable_teacache` / `enable_spectrum`** request fields are rejected with `ValueError`: every block runs at every step, so the cache heuristics do not apply. +- **`SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION`** is rejected at startup with `ValueError`: this model does not apply the offline Q/K rotation. +- **Silent output**: the reference video's audio track is kept only when it can be decoded; otherwise the output is silent and the server logs one warning per process naming the cause. A reference video without an audio track also gives a silent output (one info line). Pass `enable_audio=false` to skip audio extraction altogether. +- **Reference video not found**: `video_path` must be a path the server process can read. Pass it as `--video-path` or through `--config` offline, or in the request body online. diff --git a/docs/docs.json b/docs/docs.json index a99a0ab67d..decf40dade 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1525,7 +1525,8 @@ "group": "Wan", "pages": [ "cookbook/diffusion/Wan/Wan2.1", - "cookbook/diffusion/Wan/Wan2.2" + "cookbook/diffusion/Wan/Wan2.2", + "cookbook/diffusion/Wan/Wan-Animate-2" ] }, { diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx index 9d0d2e82b9..45d1bfc7f9 100644 --- a/docs/docs/sglang-diffusion/api/openai_api.mdx +++ b/docs/docs/sglang-diffusion/api/openai_api.mdx @@ -319,6 +319,21 @@ curl -sS -X POST "http://localhost:30010/v1/videos" \ }' ``` + +`size` is the literal output resolution for most pipelines. Wan-Animate-2 +treats it as a pixel-area budget instead: the output keeps the reference image's +aspect ratio and uses the largest frame with at most `width × height` pixels whose +sides are multiples of 16, so `size=640x800` with a 9:16 reference image yields a +528×944 video. Request a budget with the reference image's aspect ratio to get a +specific size. Wan-Animate-2 also ignores `seconds` and `num_frames`: the output +has as many frames as the (resampled) reference video, generated in chunks of +`clip_len` frames; see the +[Wan-Animate-2 cookbook](/cookbook/diffusion/Wan/Wan-Animate-2#1-2-defaults). +Wan-Animate-2 keeps the reference video's audio track on the output; the +model-specific `enable_audio` field (form field or `extra_body`, default `true`) +set to `false` gives a silent output. + + #### MP4 encoding `x264_preset` picks the libx264 preset for the returned MP4: `ultrafast`, diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx index 2bea4493f3..c35d0ee06b 100644 --- a/docs/src/snippets/diffusion/model-catalog.jsx +++ b/docs/src/snippets/diffusion/model-catalog.jsx @@ -173,6 +173,11 @@ export const DiffusionModelCatalog = ({ category }) => { ], cookbook: "/cookbook/diffusion/Wan/Wan2.2", }, + { + name: "Wan-Animate-2", + modelIds: ["Wan-AI/Wan2.2-Animate-2-14B-Diffusers"], + cookbook: "/cookbook/diffusion/Wan/Wan-Animate-2", + }, { name: "FastWan / TurboWan", modelIds: [ diff --git a/python/sglang/multimodal_gen/configs/models/dits/__init__.py b/python/sglang/multimodal_gen/configs/models/dits/__init__.py index 7b4d0fe3fa..c87cc5ab8b 100644 --- a/python/sglang/multimodal_gen/configs/models/dits/__init__.py +++ b/python/sglang/multimodal_gen/configs/models/dits/__init__.py @@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.models.dits.sana_video import SanaVideoConfig from sglang.multimodal_gen.configs.models.dits.stablediffusion3 import ( StableDiffusion3TransformerConfig, ) +from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig __all__ = [ @@ -34,6 +35,7 @@ __all__ = [ "LingBotVideoMoEConfig", "LongLive2VideoConfig", "MiniMaxH3DiTConfig", + "WanAnimate2Config", "WanVideoConfig", "Hunyuan3DDiTConfig", "MOVAAudioConfig", diff --git a/python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py b/python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py new file mode 100644 index 0000000000..c3d5ddcd54 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Wan-Animate-2 DiT config. The checkpoint is the Wan2.2-I2V-14B backbone with no +# extra parameter tensors, so WanVideoArchConfig is reused with in_channels=36, +# image_dim=1280, added_kv_proj_dim=5120 and a param-name remap. +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.base import DiTConfig +from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoArchConfig + +# Checkpoint block params use the official Wan names nested under `blocks.{i}.block.*`. +# The loader applies mapping rules in insertion order, re-scanning after each hit, so the +# extras below chain into the reused Wan lora (official -> hf) and main (hf -> native) maps. +_WAN = WanVideoArchConfig() + + +_WAN_ANIMATE_2_EXTRA_MAPPING: dict[str, str] = { + # Strip the `.block` infix so the reused Wan rules match. + r"^blocks\.(\d+)\.block\.(.*)$": r"blocks.\1.\2", + # Self-attn / cross-attn qk-norms (not covered by the Wan lora map). + r"^blocks\.(\d+)\.self_attn\.norm_q\.(.*)$": r"blocks.\1.norm_q.\2", + r"^blocks\.(\d+)\.self_attn\.norm_k\.(.*)$": r"blocks.\1.norm_k.\2", + r"^blocks\.(\d+)\.cross_attn\.norm_q\.(.*)$": r"blocks.\1.attn2.norm_q.\2", + r"^blocks\.(\d+)\.cross_attn\.norm_k\.(.*)$": r"blocks.\1.attn2.norm_k.\2", + # Cross-attn I2V image branch -> WanI2VCrossAttention add_k_proj/add_v_proj/norm_added_k. + r"^blocks\.(\d+)\.cross_attn\.k_img\.(.*)$": r"blocks.\1.attn2.add_k_proj.\2", + r"^blocks\.(\d+)\.cross_attn\.v_img\.(.*)$": r"blocks.\1.attn2.add_v_proj.\2", + r"^blocks\.(\d+)\.cross_attn\.norm_k_img\.(.*)$": r"blocks.\1.attn2.norm_added_k.\2", + # The Diffusers checkpoint keeps the official self_attn/cross_attn prefixes but names + # the projections the diffusers way (to_q/to_k/to_v/to_out.0, add_*_proj). + r"^blocks\.(\d+)\.self_attn\.to_out\.0\.(.*)$": r"blocks.\1.to_out.\2", + r"^blocks\.(\d+)\.self_attn\.(to_q|to_k|to_v)\.(.*)$": r"blocks.\1.\2.\3", + r"^blocks\.(\d+)\.cross_attn\.to_out\.0\.(.*)$": r"blocks.\1.attn2.to_out.\2", + r"^blocks\.(\d+)\.cross_attn\.(to_q|to_k|to_v|add_k_proj|add_v_proj|norm_added_k)\.(.*)$": r"blocks.\1.attn2.\2.\3", + # `norm3` is the block's only affine LayerNorm and feeds cross-attn; the reused rules + # only know its hf name (`norm2`). + r"^blocks\.(\d+)\.norm3\.(.*)$": r"blocks.\1.self_attn_residual_norm.norm.\2", + # Per-block 6-way modulation -> scale_shift_table. + r"^blocks\.(\d+)\.modulation$": r"blocks.\1.scale_shift_table", + # Output head (`head.norm` is param-free; no-op if absent). + r"^head\.head\.(.*)$": r"proj_out.\1", + r"^head\.modulation$": r"scale_shift_table", + r"^head\.norm\.(.*)$": r"norm_out.\1", + # Embedders in official naming; hf-named checkpoints are covered by the reused rules. + r"^text_embedding\.0\.(.*)$": r"condition_embedder.text_embedder.fc_in.\1", + r"^text_embedding\.2\.(.*)$": r"condition_embedder.text_embedder.fc_out.\1", + r"^time_embedding\.0\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_in.\1", + r"^time_embedding\.2\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_out.\1", + r"^time_projection\.1\.(.*)$": r"condition_embedder.time_modulation.linear.\1", + r"^img_emb\.proj\.0\.(.*)$": r"condition_embedder.image_embedder.norm1.\1", + r"^img_emb\.proj\.1\.(.*)$": r"condition_embedder.image_embedder.ff.fc_in.\1", + r"^img_emb\.proj\.3\.(.*)$": r"condition_embedder.image_embedder.ff.fc_out.\1", + r"^img_emb\.proj\.4\.(.*)$": r"condition_embedder.image_embedder.norm2.\1", + # patch_embedding, self/cross-attn q/k/v/o and ffn.* are covered by the reused Wan maps. +} + + +@dataclass +class WanAnimate2ArchConfig(WanVideoArchConfig): + """Wan2.2-I2V-14B backbone dims (inherited) with the Wan-Animate-2 I/O + remap.""" + + # Extras first, then Wan lora (official -> hf), then Wan main (hf -> native). + param_names_mapping: dict[str, str] = field( + default_factory=lambda: { + **_WAN_ANIMATE_2_EXTRA_MAPPING, + **_WAN.lora_param_names_mapping, + **_WAN.param_names_mapping, + } + ) + # Only used for re-serialization, which the inference path does not exercise. + reverse_param_names_mapping: dict[str, str] = field( + default_factory=lambda: dict(_WAN.reverse_param_names_mapping) + ) + # LoRA adapters use the official Wan names; the loader applies this before + # param_names_mapping. + lora_param_names_mapping: dict[str, str] = field( + default_factory=lambda: dict(_WAN.lora_param_names_mapping) + ) + + # in_channels: 16 noise + 20 conditioning (4-ch i2v mask + 16-ch cond latent). + # image_dim: CLIP (XLM-RoBERTa ViT-H/14) embed dim -> image_embedder. + # added_kv_proj_dim -> WanI2VCrossAttention (add_k_proj/add_v_proj/norm_added_k). + in_channels: int = 36 + out_channels: int = 16 + image_dim: int | None = 1280 + added_kv_proj_dim: int | None = 5120 + num_layers: int = 40 + + # In-context mechanism (no extra params): reference-video RoPE offsets and the flex-attention + # score_mod, consumed by forward_ref / forward_gen. The refer_* / log_scale names + # match the official and diffusers transformer configs; do not rename. In those + # names "refer" means the reference video (the motion source), not the reference image. + refer_offset_t: int = 1 + refer_offset_h: int = 0 + refer_offset_w: int = ( + -1 + ) # <0: resolved to the gen grid width in tokens at runtime (official convention) + refer_stride: int = 1 + # Additive attention-score bias on the keys of generation frame 1 (official name); + # 0.0 disables it. Read from transformer/config.json when present; the base weights omit it. + log_scale: float = 0.0 + # The image-embedding cross-attention (added_kv_proj_dim) is always built, as in the + # official model; the checkpoint config states the same and nothing else reads this. + use_img_emb: bool = True + + def __post_init__(self) -> None: + # transformer/config.json uses the official Wan names for the backbone dims; + # update_model_arch parks unknown keys in extra_attrs, so translate them here. + extras = self.extra_attrs + if "num_heads" in extras: + self.num_attention_heads = extras.pop("num_heads") + if "dim" in extras: + self.attention_head_dim = extras.pop("dim") // self.num_attention_heads + if "in_dim" in extras: + self.in_channels = extras.pop("in_dim") + if "out_dim" in extras: + self.out_channels = extras.pop("out_dim") + if not self.use_img_emb: + raise ValueError( + "Wan-Animate-2 always builds the image-embedding cross-attention; " + "a checkpoint with use_img_emb=false is a different model." + ) + self.patch_size = tuple(self.patch_size) + super().__post_init__() + + +@dataclass +class WanAnimate2Config(DiTConfig): + arch_config: WanAnimate2ArchConfig = field(default_factory=WanAnimate2ArchConfig) + prefix: str = "WanAnimate2" diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py index ddcd00feea..6ebf12c1cb 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py @@ -3,11 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 from collections.abc import Callable from dataclasses import dataclass, field +from typing import Any import torch +from sglang.multimodal_gen import envs from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig -from sglang.multimodal_gen.configs.models.dits import WanVideoConfig +from sglang.multimodal_gen.configs.models.dits import WanAnimate2Config, WanVideoConfig from sglang.multimodal_gen.configs.models.encoders import ( BaseEncoderOutput, CLIPVisionConfig, @@ -45,6 +47,10 @@ def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tenso return prompt_embeds_tensor +def animate2_postprocess_text(outputs: BaseEncoderOutput, text_inputs) -> torch.Tensor: + return outputs.last_hidden_state[0, : text_inputs["num_tokens"]] + + @dataclass class WanI2VCommonConfig(PipelineConfig): # for all wan i2v pipelines @@ -301,6 +307,127 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig): ) +@dataclass +class Wan_Animate_2_14B_Config(WanI2V720PConfig): + """Wan-Animate-2 14B (Wan2.2): reference image + reference video -> video. + + Shares the Wan I2V component bundle (T5, CLIP vision, Wan VAE with the encoder + loaded) and swaps in the in-context ``WanAnimate2Config`` DiT; a single expert, so no + ``boundary_ratio`` switching. The bespoke stages read ``flow_shift`` from here + (carrying the audio track is a per-request sampling parameter); everything else below + restates the Wan2.2 14B defaults instead of inheriting the Wan2.1 ones from the base. + """ + + # reference image + reference video -> video is I2V-shaped. + task_type: ModelTaskType = ModelTaskType.I2V + + # In-context animation DiT (in_channels=36 = 16 noise + 20 conditioning). + dit_config: DiTConfig = field(default_factory=WanAnimate2Config) + + # Denoising stage. Used when building the sigma grid. + flow_shift: float | None = 5.0 + + postprocess_text_funcs: tuple[Callable, ...] = field( + default_factory=lambda: (animate2_postprocess_text,) + ) + + def tokenize_prompt(self, prompt, tokenizer, tok_kwargs): + text_inputs = super().tokenize_prompt( + prompt, + tokenizer, + {**tok_kwargs, "max_length": self.dit_config.arch_config.text_len}, + ) + # Count on the host before TextEncodingStage uploads inputs; no GPU scalar sync. + text_inputs["num_tokens"] = int(text_inputs["attention_mask"][0].gt(0).sum()) + return text_inputs + + # The component manager casts the VAE to this dtype at its first use (the reference + # encode in the before-denoising stage), so it is the precision the whole bespoke + # VAE path runs at; fp32 like the other Wan2.2 configs changes the output. + vae_decode_precision: str = "bf16" + + def get_model_deployment_config(self) -> ModelDeploymentConfig: + # The Wan2.2 A14B modes (auto mode may layerwise-offload the 14B DiT) with the + # resident threshold the base had: at or above 60 GB free the DiT stays resident. + return ModelDeploymentConfig( + dit_layerwise_offload_modes=("auto", "memory"), + auto_dit_offload_prefetch_size=2, + keep_resident_min_available_gb=60, + keep_resident_components=("dit",), + ) + + def get_decode_scale_and_shift( + self, + device: torch.device | None, + dtype: torch.dtype | None, + vae: torch.nn.Module | None, + ) -> tuple[float, None]: + """``(1.0, None)``: ``WanAnimate2VaeAdapter.decode`` applies the latent scaling itself, + so the DecodingStage must not de-normalize again.""" + return 1.0, None + + def supports_disaggregation(self) -> bool: + # The stages share one WanAnimate2RequestState object (GPU tensors, a torch.Generator, + # the clip schedule) through batch.extra. The disaggregation transport forwards only + # JSON scalars and tensor pytrees (dict/list/tuple) from batch.extra and never the + # generator (disaggregation/extra_tensors.py, scheduler_mixin._EXCLUDE_FIELDS), so a + # split-role deployment would silently drop the state and fail at the first request. + return False + + def validate_server_args(self, server_args: Any) -> None: + if server_args.use_fsdp_inference: + raise NotImplementedError( + "Wan-Animate-2 does not support --use-fsdp-inference; use " + "--dit-layerwise-offload or --tp-size instead." + ) + # Ring splits the K/V across ranks; the in-context block mask indexes the full + # [generation | reference-video] K/V on one rank. + if server_args.ring_degree > 1: + raise NotImplementedError( + "Wan-Animate-2 does not support ring sequence parallelism " + f"(--ring-degree {server_args.ring_degree}); use --ulysses-degree N " + "with --ring-degree 1." + ) + num_heads = self.dit_config.num_attention_heads + if num_heads % server_args.tp_size != 0: + raise ValueError( + f"Wan-Animate-2 tensor parallelism needs the {num_heads} attention " + f"heads divisible by --tp-size {server_args.tp_size}." + ) + local_num_heads = num_heads // server_args.tp_size + if local_num_heads % server_args.ulysses_degree != 0: + raise ValueError( + f"Wan-Animate-2 Ulysses SP needs the TP-local head count ({local_num_heads}) " + f"divisible by --ulysses-degree {server_args.ulysses_degree}." + ) + # The in-context self-attention runs on torch flex_attention whatever backend is + # selected; only the cross-attention and the reference pass would follow the flag. + requested_dit_backend = server_args.requested_component_attention_backend( + "transformer" + ) + if ( + server_args.is_arg_explicitly_set("attention_backend") + and server_args.attention_backend is not None + ) or requested_dit_backend is not None: + raise ValueError( + "Wan-Animate-2 does not support selecting the DiT attention backend " + f"(--attention-backend {server_args.attention_backend} / " + f"--component-attention-backends transformer={requested_dit_backend}): " + "its in-context self-attention runs on torch flex_attention only. Drop " + "the flag; the text encoder still takes " + "--component-attention-backends text_encoder=." + ) + # The parent block applies the offline Q/K rotation inside its own attention + # path; forward_ref / forward_gen never do, so a rotated checkpoint would load + # and produce wrong output without a message. + if envs.SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION: + raise ValueError( + "Wan-Animate-2 does not support SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION " + "(offline Q/K rotation); unset it for this model." + ) + super().validate_server_args(server_args) + + # ============================================= # ============= Causal Self-Forcing ============= # ============================================= @@ -322,6 +449,7 @@ def register(): Wan2_2_I2V_A14B_SamplingParam, Wan2_2_T2V_A14B_SamplingParam, Wan2_2_TI2V_5B_SamplingParam, + Wan_Animate_2_14B_SamplingParam, WanI2V_14B_480P_SamplingParam, WanI2V_14B_720P_SamplingParam, WanT2V_1_3B_SamplingParams, @@ -423,3 +551,20 @@ def register(): "FastVideo/FastWan2.1-T2V-1.3B-Diffusers", ], ) + register_configs( + sampling_param_cls=Wan_Animate_2_14B_SamplingParam, + pipeline_config_cls=Wan_Animate_2_14B_Config, + hf_model_paths=[ + "Wan-AI/Wan2.2-Animate-2-14B-Diffusers", + ], + model_detectors=[ + # The registry calls this with the path and with model_index's lower-cased + # _class_name. Require the "-2-"/"_2" token so a v1 Wan2.2-Animate-14B + # checkpoint (a different model / pipeline) does not mis-route here. + lambda p: ( + "wan_animate_2" in p.lower() + or "wan2.2-animate-2" in p.lower() + or p.lower() == "wananimate2pipeline" + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/sample/wan.py b/python/sglang/multimodal_gen/configs/sample/wan.py index dfea73974d..a66976a590 100644 --- a/python/sglang/multimodal_gen/configs/sample/wan.py +++ b/python/sglang/multimodal_gen/configs/sample/wan.py @@ -1,10 +1,17 @@ # Copied and adapted from: https://github.com/hao-ai-lab/FastVideo # SPDX-License-Identifier: Apache-2.0 +import os from dataclasses import dataclass, field +from typing import Any + +import numpy as np from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) def _wan_1_3b_coefficients(p: TeaCacheParams) -> list[float]: @@ -267,6 +274,160 @@ class Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams): ) +@dataclass +class Wan_Animate_2_14B_SamplingParam(Wan2_2_Base_SamplingParams): + """Sampling defaults for Wan-Animate-2 14B.""" + + # Single expert: no guidance_scale_2 / boundary switching. + guidance_scale: float = 3.0 + num_inference_steps: int = 40 # official README demo default + + # The reference video is resampled to this rate and the output MP4 is written at it. + fps: int = 16 + # Carry the reference video's audio track on the output, as the official pipeline does: + # the track is decoded in-process and the shared MP4 writer encodes it. False gives a + # silent video. A track that cannot be decoded is logged and the output stays silent. + enable_audio: bool = True + + # Per-clip denoising length, 4k+1 (VAE stride 4); larger = better temporal coherence + # but more VRAM. Output length follows the reference video, not this. + clip_len: int = 37 + # Text for the reference-video branch (prompt_ref_embeds). The official demo conditions it on a + # fixed description of the reference video, not on the character caption. + prompt_ref: str = "人物动作的参考视频" + + # The reference image comes via the base image_path and the reference video via the + # base video_path; neither needs a model-specific field. + + # (width, height) pairs; upstream Wan-Animate default is 640x800. + supported_resolutions: list[tuple[int, int]] | None = field( + default_factory=lambda: [ + (640, 800), # upstream Wan-Animate default + (720, 1280), # 9:16 + ] + ) + + def __post_init__(self) -> None: + # The in-context denoising loop runs every block at every step; the shared DiT's + # cache heuristics are never consulted. + if self.enable_teacache or self.enable_spectrum: + raise ValueError( + "Wan-Animate-2 does not support enable_teacache or enable_spectrum." + ) + requested_clip_len = self.clip_len + self.clip_len = _round_clip_len(self.clip_len) + if self.clip_len < MIN_CLIP_LEN: + raise ValueError( + f"clip_len={requested_clip_len!r} is not supported: a clip is 4k+1 frames " + f"with k >= 1, so clip_len must be an int >= {MIN_CLIP_LEN}" + ) + if self.clip_len != requested_clip_len: + logger.info( + "clip_len=%s is not 4k+1; using clip_len=%s", + requested_clip_len, + self.clip_len, + ) + # The output length follows the reference video and the stages read clip_len, so + # `num_frames` is not a length control here; the multi-GPU frame alignment must + # leave it alone, and it must stay writable so the warmup probe can shrink. + self.adjust_frames = False + super().__post_init__() + + @classmethod + def video_request_extra_fields(cls) -> frozenset[str]: + # Settable over the /v1/videos API: the per-clip length, the reference-branch + # text (the official gradio exposes prompt_ref as well) and the audio track. The + # reference video is the base video_path field, so it needs no entry here. + return frozenset({"clip_len", "prompt_ref", "enable_audio"}) + + def prepare_synthetic_warmup_request_for_queue( + self, req: Any, server_args: Any + ) -> None: + """Give the synthetic warmup request the driving video the generic builder does not + supply, sized so the request denoises exactly one clip.""" + del server_args + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import ( + single_clip_reference_video_len, + write_reference_video, + ) + + image_path = req.image_path + if isinstance(image_path, list): + image_path = image_path[0] if image_path else None + if not isinstance(image_path, str) or not image_path: + raise ValueError( + "Wan-Animate-2 synthetic warmup requires the warmup reference image" + ) + + # The builder sizes warmup by num_frames (the class default or its bounded cap), but + # the stage reads clip_len and the compiled attention is specialized per clip + # geometry, so warm up at the production clip length unless more frames were asked. + clip_len = _round_clip_len(max(int(req.num_frames), self.clip_len)) + num_video_frames = single_clip_reference_video_len(clip_len) + fps = int(req.fps) + + # Same directory and lifetime as the warmup image; the name carries the frame count + # because every warmup request is built before the first one runs. + video_path = os.path.join( + os.path.dirname(image_path), + f"warmup_driving_video_{num_video_frames}f_{fps}fps.mp4", + ) + write_reference_video( + video_path, + _synthetic_driving_video_frames(num_video_frames), + fps=fps, + ) + + req.video_path = video_path + req.clip_len = clip_len + req.num_frames = clip_len + # The synthetic driving video has no audio track. + req.enable_audio = False + + def _adjust_visual_fields(self, server_args: Any, pipeline_config: Any) -> None: + super()._adjust_visual_fields(server_args, pipeline_config) + if self.num_frames != type(self).num_frames: + logger.info( + "Wan-Animate-2 ignores num_frames=%s (the /v1/videos `seconds`): the output " + "length follows the reference video; clip_len=%s is the per-clip chunk", + self.num_frames, + self.clip_len, + ) + + +# Smallest clip: one conditioning frame plus one VAE temporal stride (4k+1, k >= 1). +MIN_CLIP_LEN = 5 + + +def _round_clip_len(clip_len: int) -> int: + """Clips are 4k+1 pixel frames (Wan VAE temporal stride 4). Other values map to the + 4k+1 with the same k: 4k rounds up by one frame, 4k+2 and 4k+3 round down.""" + if clip_len % 4 != 1: + clip_len = clip_len // 4 * 4 + 1 + return clip_len + + +# Pixel size of the synthetic warmup driving video; the stage rescales every frame to the +# request area anyway, so this only has to be a size the shared video writer takes as is +# and cheap to encode. +_WARMUP_DRIVING_VIDEO_SIZE = 64 + + +def _synthetic_driving_video_frames(num_frames: int) -> np.ndarray: + """``[num_frames, 64, 64, 3]`` uint8 frames: a gradient that drifts one row per frame, + so consecutive frames differ like real motion.""" + size = _WARMUP_DRIVING_VIDEO_SIZE + ramp = np.linspace(0, 255, size, dtype=np.float32) + rows = np.broadcast_to(ramp[:, None], (size, size)) + cols = np.broadcast_to(ramp[None, :], (size, size)) + frames = np.empty((num_frames, size, size, 3), dtype=np.uint8) + for index in range(num_frames): + frames[index, ..., 0] = np.roll(rows, index, axis=0) + frames[index, ..., 1] = cols + frames[index, ..., 2] = 127 + return frames + + @dataclass class Turbo_Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams): guidance_scale: float = 3.5 # high_noise diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py index 7901846cb0..d9c20d19c1 100644 --- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py +++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py @@ -710,6 +710,9 @@ class WeightOverrideComponentLoader(ComponentLoader): """Base for loaders that consume an exact weights-only override.""" ignored_checkpoint_prefixes: tuple[str, ...] = () + # True: a checkpoint key that maps to no model parameter fails the load instead + # of being logged and skipped. + strict_checkpoint_keys: bool = False def load_state_dict_model( self, @@ -738,7 +741,7 @@ class WeightOverrideComponentLoader(ComponentLoader): fsdp_inference=server_args.should_use_fsdp_for_component(component_name), param_dtype=dtype, reduce_dtype=torch.float32, - strict=False, + strict=self.strict_checkpoint_keys, weight_load_plan=weight_load_plan, checkpoint_key_filter=checkpoint_key_filter, weights_iterator=weights_iterator, diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index bbf38d7a53..11daed7066 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -214,22 +214,28 @@ def fit_auto_residency_probe( floor_units = min((record.workload_units() for record in records), default=0) fitted, steps = req, 0 while True: - units = ( - max(1, int(fitted.width or 1)) - * max(1, int(fitted.height or 1)) - * max(1, int(fitted.num_frames or 1)) - ) + units = _probe_workload_units(fitted) estimate = estimate_default_workload_peak_bytes( records=records, target_units=units ) if estimate is None or estimate <= budget or units <= floor_units: return fitted, estimate, steps lighter = lighten_warmup_req(server_args, fitted) - if lighter is None: + # Sampling params that pin the frame count (Wan-Animate-2 mirrors clip_len + # in __post_init__) hand back an equal-size req; treat that as the floor. + if lighter is None or _probe_workload_units(lighter) >= units: return fitted, estimate, steps fitted, steps = lighter, steps + 1 +def _probe_workload_units(req: Req) -> int: + return ( + max(1, int(req.width or 1)) + * max(1, int(req.height or 1)) + * max(1, int(req.num_frames or 1)) + ) + + class GPUWorker(GPUWorkerPostTrainingMixin): """ A worker that executes the model on a single GPU. diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py new file mode 100644 index 0000000000..9e55ca5ab1 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py @@ -0,0 +1,445 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Wan-Animate-2 DiT. Subclasses WanTransformer3DModel so TP / Ulysses SP shard it +# unchanged; only the per-block self-attention is swapped for the in-context mechanism +# (build_reference_kv runs forward_ref once per clip and returns each block's rotated reference-video +# K/V, forward_gen attends over [gen tokens, then that reference-video K/V]). No extra parameters. +# Names kept from the reference implementation: the forward_ref / forward_gen methods and the +# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image. +from collections import OrderedDict +from typing import Any + +import torch +import torch.nn as nn +from torch.nn.attention.flex_attention import BlockMask, create_block_mask + +from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import ( + WanAnimate2Config, +) +from sglang.multimodal_gen.runtime.distributed import get_sp_world_size +from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import ( + gather_seq, + shard_seq, +) +from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import ( + QuantizationConfig, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import ( + WanAnimate2TransformerBlock, + _InContextLayout, + _reference_video_key_mask, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import ( + WanAnimate2ClipConditioning, + WanAnimate2ReferenceKV, +) +from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformer3DModel +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +class WanAnimate2Transformer3DModel(WanTransformer3DModel): + """Wan-Animate-2 DiT (in-context variant) on the Wan2.2-I2V-14B backbone. + + Attention goes through USPAttention, so callers must wrap the forward in + ``set_forward_context(...)``. + """ + + # Registry alias: the diffusers class name of this transformer. + _aliases = ["WanAnimate2Transformer3DModel"] + + param_names_mapping = WanAnimate2Config().param_names_mapping + reverse_param_names_mapping = WanAnimate2Config().reverse_param_names_mapping + lora_param_names_mapping = WanAnimate2Config().lora_param_names_mapping + + def __init__( + self, + config: WanAnimate2Config, + hf_config: dict[str, Any] | None = None, + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__( + config=config, hf_config=hf_config or {}, quant_config=quant_config + ) + + # RoPE placement of the reference-video tokens relative to the gen grid; plain + # ints from the config, not parameters, so the Wan state dict is unchanged. + self.refer_offset_t = config.refer_offset_t + self.refer_offset_h = config.refer_offset_h + # <0: place reference-video tokens right of the gen grid (w offset = gen W in tokens). + self.refer_offset_w = config.refer_offset_w + self.refer_stride = config.refer_stride + # Prompt embeddings are padded/truncated to this many tokens. + self.max_text_len = config.text_len + + self._block_masks: OrderedDict[tuple[int, int, int, str], BlockMask] = ( + OrderedDict() + ) + # RoPE tables keyed by the (f, h, w) position ranges and device; see _rope_tables. + self._rope_cache: OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]] = ( + OrderedDict() + ) + + def _make_block( + self, + *, + index: int, + config: WanAnimate2Config, + quant_config: QuantizationConfig | None, + ) -> nn.Module: + # Same constructor args as the Wan block and no new params, so the Wan checkpoint + # mapping and TP sharding apply unchanged. + return WanAnimate2TransformerBlock( + config.num_attention_heads * config.attention_head_dim, + config.ffn_dim, + config.num_attention_heads, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + self._supported_attention_backends + | {AttentionBackendEnum.VIDEO_SPARSE_ATTN}, + prefix=f"blocks.{index}", + attention_type=config.attention_type, + sla_topk=config.sla_topk, + quant_config=quant_config, + log_scale=config.log_scale, + ) + + # Device and dtype of the resident weights; the patch embedding is never offloaded. + @property + def _device(self) -> torch.device: + return self.patch_embedding.proj.weight.device + + @property + def _dtype(self) -> torch.dtype: + return self.patch_embedding.proj.weight.dtype + + def _rope_tables( + self, + f_range: tuple[int, int, int], # (start, length, stride) + h_range: tuple[int, int], # (start, length) + w_range: tuple[int, int], + ) -> tuple[torch.Tensor, torch.Tensor]: + """RoPE ``(cos, sin)``, each ``[f*h*w, D//2]`` fp32, for the full grid. + + Row i holds the position of token i, flattened in (f, h, w) row-major order + like the patch embedding output. + + The table is cached per ranges.""" + key = (f_range, h_range, w_range, str(self._device)) + if key in self._rope_cache: + self._rope_cache.move_to_end(key) + return self._rope_cache[key] + # Bound retained GPU tables; evict before allocating the next geometry. + if len(self._rope_cache) >= 4: + self._rope_cache.popitem(last=False) + f_start, f_len, f_stride = f_range + f_idx = torch.arange(f_len, device=self._device) * f_stride + f_start + h_idx = torch.arange(h_range[1], device=self._device) + h_range[0] + w_idx = torch.arange(w_range[1], device=self._device) + w_range[0] + # Positions are built explicitly (not via the rank-sharding forward_from_grid) because + # the in-context blocks all-to-all-gather the full sequence before applying RoPE. + ff, hh, ww = torch.meshgrid(f_idx, h_idx, w_idx, indexing="ij") + positions = torch.stack([ff.reshape(-1), hh.reshape(-1), ww.reshape(-1)], dim=1) + cos, sin = self.rotary_emb.forward_uncached(positions.long()) + tables = (cos.float(), sin.float()) + self._rope_cache[key] = tables + return tables + + def _generation_video_rope( + self, f: int, h: int, w: int + ) -> tuple[torch.Tensor, torch.Tensor]: + """RoPE tables for the generated video's ``f x h x w`` token grid; positions start at 0.""" + return self._rope_tables((0, f, 1), (0, h), (0, w)) + + def _reference_video_rope( + self, f: int, h: int, w: int, generation_video_w: int + ) -> tuple[torch.Tensor, torch.Tensor]: + """RoPE tables for the reference video's ``f x h x w`` token grid, shifted by the + ``refer_*`` offsets. When ``refer_offset_w`` is -1, the reference-video grid is placed + directly to the right of the generated grid (width offset = ``generation_video_w``).""" + offset_w = ( + generation_video_w if self.refer_offset_w == -1 else self.refer_offset_w + ) + return self._rope_tables( + (self.refer_offset_t, f, self.refer_stride), + (self.refer_offset_h, h), + (offset_w, w), + ) + + def _get_block_mask_from_layout(self, layout: _InContextLayout) -> BlockMask: + """Cached flex_attention ``BlockMask`` over the padded [gen | reference-video] layout.""" + key = ( + layout.max_generation_video_num_tokens, + layout.max_reference_video_num_tokens, + layout.max_tokens_per_frame, + str(self._device), + ) + if key in self._block_masks: + self._block_masks.move_to_end(key) + return self._block_masks[key] + if len(self._block_masks) >= 2: + self._block_masks.popitem(last=False) + + # _compile=True keeps the block-sparse layout consistent with the compiled + # flex_attention; built once per key, so the compile cost is paid once. + block_mask = create_block_mask( + layout.block_mask_fn(), + B=None, + H=None, + Q_LEN=layout.padded_generation_video_len, + KV_LEN=layout.padded_kv_len, + device=self._device, + _compile=True, + ) + self._block_masks[key] = block_mask + return block_mask + + def _patch_embeddings( + self, + latent_36ch: torch.Tensor, # [B, 36, f, H, W] + ) -> tuple[torch.Tensor, tuple[int, int, int]]: + """Patch-embed the latent into the token stream ``[B, f*h*w, C]`` and return the + token grid_size ``(f, h, w)``; tokens are flattened row-major, w fastest.""" + p_t, p_h, p_w = self.patch_size + _, _, num_frames, height, width = latent_36ch.shape + grid_size = (num_frames // p_t, height // p_h, width // p_w) + hidden_states = self.patch_embedding(latent_36ch) + hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous() + return hidden_states, grid_size + + def _time_embeddings( + self, timestep: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + timestep_embeddings = self.condition_embedder.time_embedder(timestep, None) + timestep_modulation = self.condition_embedder.time_modulation( + timestep_embeddings + ).unflatten(1, (6, -1)) + return timestep_embeddings, timestep_modulation + + def prepare_context( + self, + prompt_embeddings: torch.Tensor, + image_embeddings: torch.Tensor | None, + ) -> torch.Tensor: + """Project one branch's timestep-invariant context; owned by the current clip.""" + prompt_embeddings = self._pad_or_truncate_prompt_embeddings(prompt_embeddings) + context = self.condition_embedder.text_embedder(prompt_embeddings) + if image_embeddings is not None: + image_context = self.condition_embedder.image_embedder(image_embeddings) + context = torch.cat([image_context, context], dim=1) + return context.to(self._dtype) + + def _pad_or_truncate_prompt_embeddings( + self, prompt_embeddings: torch.Tensor + ) -> torch.Tensor: + """Zero-pad or truncate the prompt embeddings (``[L, C]`` or ``[B, L, C]``) to + ``[B, max_text_len, C]``, as the official model does; cross-attention attends the pad rows too.""" + if prompt_embeddings.dim() == 2: + prompt_embeddings = prompt_embeddings.unsqueeze(0) # [L, C] -> [1, L, C] + + batch_size, num_tokens, channels = prompt_embeddings.shape + if num_tokens < self.max_text_len: + pad = prompt_embeddings.new_zeros( + batch_size, self.max_text_len - num_tokens, channels + ) + prompt_embeddings = torch.cat([prompt_embeddings, pad], dim=1) + elif num_tokens > self.max_text_len: + prompt_embeddings = prompt_embeddings[:, : self.max_text_len] + return prompt_embeddings.to(self._dtype) + + @torch.no_grad() + def build_reference_kv( + self, *, clip_cond: WanAnimate2ClipConditioning + ) -> WanAnimate2ReferenceKV: + """One forward_ref pass over the clip's reference video, returning every block's rotated K and V. + + Text- and timestep-independent, so one result serves every step and CFG branch of the + clip; the caller owns it and the DiT keeps no copy. Attention goes through USPAttention, + so callers must wrap this in ``set_forward_context(...)`` like ``forward``. + """ + + # 36-channel reference-video input: fp32 latents (16) over the bf16 condition (20), + # both cast to the param dtype so the cat runs in bf16 (see _make_generation_input). + reference_video_latent_36ch = torch.cat( + [ + clip_cond.reference_video_latents.to(self._dtype), + clip_cond.reference_video_condition.to(self._dtype), + ], + dim=0, + ).unsqueeze(0) # [1, 36, cf, H', W'] + reference_video_frame_0_image_embeddings = ( + clip_cond.reference_video_frame_0_image_embeddings + ) # [1, 257, 1280] bf16 + + # reference-video timestep is forced to 1. + reference_video_timestep = torch.ones(1, device=self._device, dtype=torch.long) + hidden_states, (reference_video_f, reference_video_h, reference_video_w) = ( + self._patch_embeddings(reference_video_latent_36ch) + ) + _, timestep_modulation = self._time_embeddings(reference_video_timestep) + reference_video_encoder_hidden_states = self.prepare_context( + clip_cond.prompt_ref_embeddings, reference_video_frame_0_image_embeddings + ) + + reference_video_rope_cos, reference_video_rope_sin = self._reference_video_rope( + reference_video_f, + reference_video_h, + reference_video_w, + generation_video_w=clip_cond.generation_video_grid_sizes[2], + ) + reference_video_num_tokens = ( + reference_video_f * reference_video_h * reference_video_w + ) + + # Ulysses SP: shard the reference-video sequence; each block a2a-gathers it and caches + # full-seq / head-shard K/V. The reference-video hidden states are discarded, so no gather. + # The gathered sequence is tail-padded to a multiple of the SP size; one key mask over + # that padding serves every block (None when nothing is padded). + reference_video_key_mask = None + sp_size = get_sp_world_size() + if sp_size > 1: + hidden_states, _ = shard_seq(hidden_states, dim=1) + gathered_seq_len = hidden_states.shape[1] * sp_size + if gathered_seq_len != reference_video_num_tokens: + reference_video_key_mask = _reference_video_key_mask( + batch_size=hidden_states.shape[0], + seq_len=gathered_seq_len, + num_valid=reference_video_num_tokens, + device=hidden_states.device, + ) + + k_by_block: dict[int, torch.Tensor] = {} + v_by_block: dict[int, torch.Tensor] = {} + # Call via __call__ (not forward_ref) so the layerwise-offload hooks fire. + for idx, block in enumerate(self.blocks): + hidden_states = block( + kv_cache_mode="ref", + hidden_states=hidden_states, + encoder_hidden_states=reference_video_encoder_hidden_states, + timestep_modulation=timestep_modulation, + reference_video_rope_cos=reference_video_rope_cos, + reference_video_rope_sin=reference_video_rope_sin, + reference_video_num_tokens=reference_video_num_tokens, + index=idx, + k_cache=k_by_block, + v_cache=v_by_block, + reference_video_key_mask=reference_video_key_mask, + ) + return WanAnimate2ReferenceKV(k=k_by_block, v=v_by_block) + + @torch.no_grad() + def forward( + self, + hidden_states: torch.Tensor, # [16, f, h, w] gen latent (B=1) + encoder_hidden_states: torch.Tensor, # this CFG branch's prompt embeddings [L, C] + timestep: torch.Tensor, # [1] + # [1, 257, 1280] bf16 CLIP embed of the reference image + encoder_hidden_states_image: torch.Tensor | None = None, + guidance: torch.Tensor | None = None, + *, + # This clip's K/V from build_reference_kv; the same object for every step and CFG branch. + reference_kv: WanAnimate2ReferenceKV, + # This clip's conditioning from build_clip_conditioning. + clip_cond: WanAnimate2ClipConditioning, + projected_context: torch.Tensor | None = None, + # The unconditional CFG branch skips block 9 (see _run_generation_blocks). + is_unconditional: bool = False, + ) -> torch.Tensor: + sp_size = get_sp_world_size() + + latent_36ch = self._make_generation_input( + hidden_states, clip_cond.generation_condition + ) + hidden_states, (f, h, w) = self._patch_embeddings(latent_36ch) + timestep_embeddings, timestep_modulation = self._time_embeddings(timestep) + generation_encoder_hidden_states = ( + projected_context + if projected_context is not None + else self.prepare_context( + encoder_hidden_states, encoder_hidden_states_image + ) + ) + layout = _InContextLayout( + (f, h, w), + clip_cond.reference_video_grid_sizes, + clip_cond.full_clip_grid_sizes, + ) + + # No seq padding here: hidden_states is exactly f*h*w tokens and the in-context + # layout pads internally. Under Ulysses SP each block a2a-gathers the full + # sequence, so shard tail padding lands at the global tail where forward_gen + # expects it. + if sp_size > 1: + hidden_states, _ = shard_seq(hidden_states, dim=1) + hidden_states = self._run_generation_blocks( + hidden_states, + encoder_hidden_states=generation_encoder_hidden_states, + timestep_modulation=timestep_modulation, + layout=layout, + reference_kv=reference_kv, + is_unconditional=is_unconditional, + ) + if sp_size > 1: + # Gather the shards and trim tail padding back to the clip's token count. + hidden_states = gather_seq( + hidden_states, layout.generation_video_num_tokens, dim=1 + ) + + return self._unpatchify(hidden_states, timestep_embeddings, (f, h, w)) + + def _make_generation_input( + self, + latent: torch.Tensor, # [16, f, h, w] fp32 + generation_condition: torch.Tensor, # [20, f, h, w] bf16 + ) -> torch.Tensor: + """36-channel DiT input ``[1, 36, f, h, w]``: the fp32 noisy latent over the bf16 generation condition.""" + # Both cast to the param dtype so the cat runs in bf16 and does not rely on autocast; + # the fp32 latent would be rounded to bf16 by the patch embedding anyway. + return torch.cat( + [latent.to(self._dtype), generation_condition.to(self._dtype)], dim=0 + ).unsqueeze(0) + + def _run_generation_blocks( + self, + hidden_states: torch.Tensor, + *, + encoder_hidden_states: torch.Tensor, + timestep_modulation: torch.Tensor, + layout: _InContextLayout, + reference_kv: WanAnimate2ReferenceKV, + is_unconditional: bool, + ) -> torch.Tensor: + """forward_gen through every block with the in-context RoPE tables and block mask.""" + generation_video_rope_cos, generation_video_rope_sin = ( + self._generation_video_rope( + layout.generation_video_f, + layout.generation_video_h, + layout.generation_video_w, + ) + ) + block_mask = self._get_block_mask_from_layout(layout) + + # Call via __call__ (not forward_gen) so the layerwise-offload hooks fire. + for idx, block in enumerate(self.blocks): + # Block 9 is dropped on the unconditional branch, as in the reference + # wan_animate_2_model.forward_gen; the authors give no rationale. + if is_unconditional and idx == 9: + continue + hidden_states = block( + kv_cache_mode="gen", + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep_modulation=timestep_modulation, + gen_cos=generation_video_rope_cos, + gen_sin=generation_video_rope_sin, + block_mask=block_mask, + layout=layout, + index=idx, + k_cache=reference_kv.k, + v_cache=reference_kv.v, + ) + return hidden_states + + +EntryClass = WanAnimate2Transformer3DModel diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py new file mode 100644 index 0000000000..2c906f3076 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py @@ -0,0 +1,532 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Wan-Animate-2 DiT block: in-context self-attention over [gen tokens, then cached +# reference-video K/V], its RoPE / Ulysses helpers, and the per-clip _InContextLayout +# geometry. +# Names kept from the reference implementation: the forward_ref / forward_gen methods and the +# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image. +from __future__ import annotations + +import math +from collections.abc import Callable +from functools import lru_cache, partial +from typing import Any, Literal + +import torch +from torch.nn.attention.flex_attention import BlockMask, flex_attention + +from sglang.multimodal_gen.runtime.distributed import ( + get_sp_world_size, +) +from sglang.multimodal_gen.runtime.layers.usp import ( + _usp_input_all_to_all_qkv as _sp_gather_qkv, +) +from sglang.multimodal_gen.runtime.layers.usp import ( + _usp_output_all_to_all, +) +from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformerBlock + +# Eager flex_attention materializes the full [Q, KV] scores over the padded [gen | reference-video] +# sequence (tens of GiB at video resolution); compiled, it is one fused kernel. +_flex_attention_compiled = torch.compile( + flex_attention, + dynamic=False, + mode="max-autotune", + fullgraph=True, + backend="inductor", +) + + +def _apply_rope_interleaved( + x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor +) -> torch.Tensor: + """Interleaved (GPT-J style) RoPE, identical to ``_apply_rotary_emb(is_neox_style=False)``; + ``x`` is ``[B, S, H, D]``, ``cos`` / ``sin`` are ``[S, D//2]`` fp32.""" + cos = cos[None, :, None, :] # [1, S, 1, D//2] + sin = sin[None, :, None, :] + x1 = x[..., 0::2].float() + x2 = x[..., 1::2].float() + o1 = x1 * cos - x2 * sin + o2 = x2 * cos + x1 * sin + out = torch.stack([o1, o2], dim=-1).flatten(-2) + return out.type_as(x) + + +def _score_mod_impl( + score: torch.Tensor, + b_idx: torch.Tensor, + h_idx: torch.Tensor, + q_idx: torch.Tensor, + kv_idx: torch.Tensor, + hw: int, + log_scale: float, +) -> torch.Tensor: + # Biases the keys of generation frame 1, [hw, 2*hw) in the padded layout, exactly as the + # official Wan-Animate-2 code does; non-zero only when the checkpoint config sets log_scale. + condition = (kv_idx >= hw) & (kv_idx < 2 * hw) + return torch.where(condition, score + log_scale, score) + + +@lru_cache(maxsize=None) +def _make_score_mod(hw: int, log_scale: float) -> Callable[..., torch.Tensor]: + # Stable object identity per (hw, log_scale): compiled flex_attention guards on the + # score_mod object, so a fresh closure per call would recompile every clip. + return partial(_score_mod_impl, hw=hw, log_scale=log_scale) + + +def _sp_scatter_out(attn_output: torch.Tensor) -> torch.Tensor: + """Ulysses output a2a: [B, S_full, N_local, C] -> [B, S_local, N_tp, C].""" + return _usp_output_all_to_all(attn_output, head_dim=2) + + +def _reference_video_key_mask( + *, batch_size: int, seq_len: int, num_valid: int, device: torch.device +) -> torch.Tensor: + """``[B, S]`` bool key mask: True for the leading ``num_valid`` reference-video tokens.""" + key_mask = torch.zeros((batch_size, seq_len), dtype=torch.bool, device=device) + key_mask[:, :num_valid] = True + return key_mask + + +class WanAnimate2TransformerBlock(WanTransformerBlock): + """Wan block with the Wan-Animate-2 in-context self-attention. + + No new parameters: forward_ref / forward_gen reuse the inherited modules and only + route the self-attention differently. + """ + + def __init__(self, *args: Any, log_scale: float = 0.0, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + if self.use_offline_qk_rotation: + raise ValueError( + "Wan-Animate-2 blocks do not apply the offline Q/K rotation that " + "SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION selects for this checkpoint." + ) + # Additive score bias on the keys of generation frame 1 (_score_mod_impl); 0.0 skips it. + self.log_scale = log_scale + + def _modulate( + self, timestep_modulation: torch.Tensor + ) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + ]: + """14B modulation split of ``timestep_modulation`` ``[B, 6, C]`` into six ``[B, 1, C]`` fp32 chunks; mirrors wanvideo.py.""" + assert timestep_modulation.dim() == 3, ( + "WanAnimate2TransformerBlock expects the 14B timestep modulation [B, 6, inner]; the ti2v " + "seq-len modulation path is not used by Wan-Animate-2." + ) + e = self.scale_shift_table + timestep_modulation.float() + shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk( + 6, dim=1 + ) + return shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa + + def forward( + self, *, kv_cache_mode: Literal["ref", "gen"], **kwargs: Any + ) -> torch.Tensor: + """Dispatch to forward_ref / forward_gen through ``nn.Module.__call__`` so the + layerwise-offload module hooks fire for both passes; keyword arguments only.""" + if kv_cache_mode == "ref": + return self.forward_ref(**kwargs) + if kv_cache_mode == "gen": + return self.forward_gen(**kwargs) + raise ValueError( + f"WanAnimate2TransformerBlock: unknown kv_cache_mode {kv_cache_mode!r} " + "(expected 'ref' or 'gen')." + ) + + def forward_ref( + self, + *, + hidden_states: torch.Tensor, # [B, S_reference_video, C] reference-video latent stream + encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] reference-video context (CLIP tokens, then text) + timestep_modulation: torch.Tensor, # [B, 6, C], reference-video timestep forced to 1 + reference_video_rope_cos: torch.Tensor, # reference-video RoPE cos [reference_video_num_tokens, D//2] + reference_video_rope_sin: torch.Tensor, + reference_video_num_tokens: int, # reference_video_f * reference_video_h * reference_video_w (valid reference-video tokens) + index: int, + k_cache: dict[int, torch.Tensor], + v_cache: dict[int, torch.Tensor], + # [B, S_reference_video] key mask for the SP tail padding, built once per clip by the + # caller; None when the gathered sequence has no padding. + reference_video_key_mask: torch.Tensor | None = None, + ) -> torch.Tensor: + """Cache this block's rotated reference-video K and V, then run the reference-video self-attention + (reference-video RoPE, pad keys masked) and return the updated reference-video stream.""" + orig_dtype = hidden_states.dtype + shift, scale, gate, c_shift, c_scale, c_gate = self._modulate( + timestep_modulation + ) + + norm_hidden_states = self.norm1(hidden_states, shift, scale) + query, key, value = self._qkv(norm_hidden_states, self.local_num_heads) + + # Ulysses SP: gather the full reference-video sequence / shard heads before caching, so the + # cache matches forward_gen's post-a2a layout. + sp_size = get_sp_world_size() + if sp_size > 1: + query, key, value = _sp_gather_qkv(query, key, value) + + # Reference-video self-attention. Under SP, S_reference_video is the gathered + # (tail-padded) length: RoPE the valid slice and mask the pad keys. Without padding the + # whole sequence is valid and no mask is passed: USPAttention serves a dense key mask + # through SDPA instead of the configured backend. + reference_video_seq_len = query.shape[1] + if reference_video_seq_len == reference_video_num_tokens: + q_roped = _apply_rope_interleaved( + query, reference_video_rope_cos, reference_video_rope_sin + ) + k_roped = _apply_rope_interleaved( + key, reference_video_rope_cos, reference_video_rope_sin + ) + key_mask = None + else: + q_roped = query.clone() + k_roped = key.clone() + q_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved( + query[:, :reference_video_num_tokens], + reference_video_rope_cos, + reference_video_rope_sin, + ) + k_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved( + key[:, :reference_video_num_tokens], + reference_video_rope_cos, + reference_video_rope_sin, + ) + key_mask = reference_video_key_mask + if key_mask is None: + key_mask = _reference_video_key_mask( + batch_size=hidden_states.shape[0], + seq_len=reference_video_seq_len, + num_valid=reference_video_num_tokens, + device=hidden_states.device, + ) + # Reference positions are invariant within a clip. Keep the rotated key in the + # original cache storage; SP's value can share the gathered QKV allocation. + k_cache[index] = key.copy_(k_roped) + v_cache[index] = value + if sp_size > 1: + # q/k/v are already gathered: skip USPAttention's own a2a, then scatter back. + attn_output = self.attn1( + q_roped, + k_roped, + value, + attn_mask=key_mask, + skip_sequence_parallel_override=True, + ) + attn_output = _sp_scatter_out(attn_output) + else: + attn_output = self.attn1(q_roped, k_roped, value, attn_mask=key_mask) + attn_output = attn_output.flatten(2) + + return self._cross_and_ffn( + hidden_states, + attn_output, + encoder_hidden_states, + gate, + c_shift, + c_scale, + c_gate, + orig_dtype, + ) + + def forward_gen( + self, + *, + hidden_states: torch.Tensor, # [B, seq_len, C] gen stream (padded) + encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] gen context (CLIP tokens, then text) + timestep_modulation: torch.Tensor, # [B, 6, C] + gen_cos: torch.Tensor, # gen-RoPE cos [generation_video_num_tokens, D//2] + gen_sin: torch.Tensor, + block_mask: BlockMask, # per full-length clip grid + layout: _InContextLayout, + index: int, + k_cache: dict[int, torch.Tensor], + v_cache: dict[int, torch.Tensor], + ) -> torch.Tensor: + """One generation block with in-context masked self-attention over + [gen tokens, then cached reference-video].""" + orig_dtype = hidden_states.dtype + shift, scale, gate, c_shift, c_scale, c_gate = self._modulate( + timestep_modulation + ) + + norm_hidden_states = self.norm1(hidden_states, shift, scale) + query, key, value = self._qkv(norm_hidden_states, self.local_num_heads) + + # Ulysses SP: gather the full gen sequence / shard heads. The cached reference-video K/V are + # already in this layout. + sp_size = get_sp_world_size() + if sp_size > 1: + query, key, value = _sp_gather_qkv(query, key, value) + + # gen-RoPE on the valid gen tokens only (padding kept unrotated). + num_generation_tokens = layout.generation_video_num_tokens + q_roped = _apply_rope_interleaved( + query[:, :num_generation_tokens], gen_cos, gen_sin + ) + k_roped = _apply_rope_interleaved( + key[:, :num_generation_tokens], gen_cos, gen_sin + ) + if query.shape[1] != num_generation_tokens: + q_roped = torch.cat([q_roped, query[:, num_generation_tokens:]], dim=1) + k_roped = torch.cat([k_roped, key[:, num_generation_tokens:]], dim=1) + + # Reference-video K is already rotated by forward_ref, including SP tail padding. + reference_video_k = k_cache[index] + reference_video_v = v_cache[index] + + attn_output = self._in_context_attention( + q_roped, + k_roped, + value, + reference_video_k, + reference_video_v, + block_mask, + layout, + ) + if sp_size > 1: + # scatter the sequence back / gather heads. + attn_output = _sp_scatter_out(attn_output) + attn_output = attn_output.flatten(2) + + return self._cross_and_ffn( + hidden_states, + attn_output, + encoder_hidden_states, + gate, + c_shift, + c_scale, + c_gate, + orig_dtype, + ) + + def _in_context_attention( + self, + q: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd) + k: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd) + v: torch.Tensor, # [B, seq_len, H, D] + reference_video_k: torch.Tensor, # [B, reference_video_seq_len, H, D] (reference-video RoPE'd) + reference_video_v: torch.Tensor, # [B, reference_video_seq_len, H, D] + block_mask: BlockMask, + layout: _InContextLayout, + ) -> torch.Tensor: + """Build the padded [gen | reference-video] layout and run compiled flex_attention. + + The block-mask / score_mod are over sequence positions only, so TP composes + trivially, and under Ulysses SP the callers hand in the full sequence with sharded + heads, so the layout and mask are identical to sp=1.""" + batch_size, _, num_heads, head_dim = q.shape + device, dtype = q.device, q.dtype + padded_generation_video_len = layout.padded_generation_video_len + padded_kv_len = layout.padded_kv_len + + q_padding = q[:, layout.generation_video_num_tokens :] + + q_padded = torch.zeros( + batch_size, + padded_generation_video_len, + num_heads, + head_dim, + device=device, + dtype=dtype, + ) + k_padded = torch.zeros( + batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype + ) + v_padded = torch.zeros( + batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype + ) + + num_latent_frames = layout.generation_video_f + latent_hw = layout.generation_video_tokens_per_frame + q_valid = q[:, : layout.generation_video_num_tokens].view( + batch_size, num_latent_frames, latent_hw, num_heads, head_dim + ) + k_valid = k[:, : layout.generation_video_num_tokens].view( + batch_size, num_latent_frames, latent_hw, num_heads, head_dim + ) + v_valid = v[:, : layout.generation_video_num_tokens].view( + batch_size, num_latent_frames, latent_hw, num_heads, head_dim + ) + max_tokens_per_frame = layout.max_tokens_per_frame + q_padded[:, : num_latent_frames * max_tokens_per_frame].view( + batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim + )[:, :, :latent_hw] = q_valid + k_padded[:, : num_latent_frames * max_tokens_per_frame].view( + batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim + )[:, :, :latent_hw] = k_valid + v_padded[:, : num_latent_frames * max_tokens_per_frame].view( + batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim + )[:, :, :latent_hw] = v_valid + + reference_video_f, reference_video_tokens_per_frame = ( + layout.reference_video_f, + layout.reference_video_tokens_per_frame, + ) + reference_video_k_valid = reference_video_k[ + :, : layout.reference_video_num_tokens + ].view( + batch_size, + reference_video_f, + reference_video_tokens_per_frame, + num_heads, + head_dim, + ) + reference_video_v_valid = reference_video_v[ + :, : layout.reference_video_num_tokens + ].view( + batch_size, + reference_video_f, + reference_video_tokens_per_frame, + num_heads, + head_dim, + ) + k_padded[ + :, + padded_generation_video_len : padded_generation_video_len + + reference_video_f * max_tokens_per_frame, + ].view( + batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim + )[:, :, :reference_video_tokens_per_frame] = reference_video_k_valid + v_padded[ + :, + padded_generation_video_len : padded_generation_video_len + + reference_video_f * max_tokens_per_frame, + ].view( + batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim + )[:, :, :reference_video_tokens_per_frame] = reference_video_v_valid + + # At 0.0 the bias is an exact identity, so the kernel is not asked to apply it; + # otherwise the cached partial keeps the identity compiled flex guards on (_make_score_mod). + log_scale = float(self.log_scale) + _score_mod = ( + None + if log_scale == 0.0 + else _make_score_mod(max_tokens_per_frame, log_scale) + ) + + # flex_attention takes [B, H, L, D]; the default 1/sqrt(D) scale matches USPAttention. + attn_output_padded = _flex_attention_compiled( + q_padded.transpose(1, 2), + k_padded.transpose(1, 2), + v_padded.transpose(1, 2), + block_mask=block_mask, + score_mod=_score_mod, + ).transpose(1, 2) # -> [B, padded_generation_video_len, N, C] + + attn_output_valid = attn_output_padded[ + :, : num_latent_frames * max_tokens_per_frame + ].view( + batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim + )[:, :, :latent_hw] + attn_output_valid = attn_output_valid.reshape( + batch_size, num_latent_frames * latent_hw, num_heads, head_dim + ) + if q_padding.shape[1] == 0: + return attn_output_valid + return torch.cat([attn_output_valid, q_padding], dim=1) # [B, seq_len, N, C] + + +class _InContextLayout: + """Token geometry of the padded [generation | reference-video] in-context layout. + + Three sets of sizes: this clip's token counts; those of a full-length clip, which every + clip is padded up to; and the full-length sizes rounded up to 128 for flex_attention. + The attention pads to the last set and the block mask is built from the same numbers, + so one compiled mask serves every clip of a request.""" + + def __init__( + self, + generation_video_grid: tuple[int, int, int], + reference_video_grid: tuple[int, int, int], + full_clip_grid: tuple[int, int, int], + ) -> None: + f, h, w = generation_video_grid + reference_video_f, reference_video_h, reference_video_w = reference_video_grid + full_clip_f, full_clip_h, full_clip_w = full_clip_grid + self.generation_video_f, self.generation_video_h, self.generation_video_w = ( + f, + h, + w, + ) + self.reference_video_f, self.reference_video_h, self.reference_video_w = ( + reference_video_f, + reference_video_h, + reference_video_w, + ) + + # This clip; shorter than full length only for the last clip of a video. + self.generation_video_tokens_per_frame = h * w + self.reference_video_tokens_per_frame = reference_video_h * reference_video_w + self.generation_video_num_tokens = f * h * w + self.reference_video_num_tokens = ( + reference_video_f * reference_video_h * reference_video_w + ) + + # A full-length clip. The generation video carries one extra reference-image frame, + # the reference video does not. + self.max_tokens_per_frame = full_clip_h * full_clip_w + self.max_generation_video_num_tokens = full_clip_f * self.max_tokens_per_frame + self.max_reference_video_num_tokens = ( + full_clip_f - 1 + ) * self.max_tokens_per_frame + + # Padded to flex_attention's 128-token blocks; keys are [generation | reference video]. + self.padded_generation_video_len = ( + math.ceil(self.max_generation_video_num_tokens / 128) * 128 + ) + self.padded_reference_video_len = ( + math.ceil(self.max_reference_video_num_tokens / 128) * 128 + ) + self.padded_kv_len = ( + self.padded_generation_video_len + self.padded_reference_video_len + ) + + def block_mask_fn(self) -> Callable[..., torch.Tensor]: + """Returns the block mask function based on the [gen | reference-video] in-context layout. + + A valid gen query attends every valid gen key plus the reference-video keys of its own frame. + + keys: [generation tokens | generation pad | reference tokens | reference pad] + 0 padded_generation_video_len padded_kv_len + queries: [generation tokens | generation pad] + 0 padded_generation_video_len + """ + tokens_per_frame = self.max_tokens_per_frame + num_generation_tokens = self.max_generation_video_num_tokens + num_reference_tokens = self.max_reference_video_num_tokens + reference_start = self.padded_generation_video_len + + def _mask_function( + b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor + ) -> torch.Tensor: + # Pad positions are not tokens; pad queries attend nothing. + q_is_generation_token = q_idx < num_generation_tokens + # Generation frame of the query; frame 0 is the reference image. + q_frame = q_idx // tokens_per_frame + + # Every generation token attends every generation token. + kv_is_generation_token = kv_idx < num_generation_tokens + + # Reference-video tokens are attended only by the queries of their own frame. + kv_reference_idx = kv_idx - reference_start + kv_is_reference_token = (kv_idx >= reference_start) & ( + kv_reference_idx < num_reference_tokens + ) + # +1: the reference video has no counterpart for the generation stream's + # reference-image frame 0. + kv_reference_frame = kv_reference_idx // tokens_per_frame + 1 + kv_is_same_frame_reference_token = kv_is_reference_token & ( + q_frame == kv_reference_frame + ) + return q_is_generation_token & ( + kv_is_generation_token | kv_is_same_frame_reference_token + ) + + return _mask_function diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py new file mode 100644 index 0000000000..ec2e4d662e --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Per-clip contracts between the Wan-Animate-2 stages and the DiT. + +Lives on the model side so the DiT can type its ``clip_cond`` / ``reference_kv`` kwargs +without importing a pipeline stage: the before-denoising stage builds the conditioning, the +DiT's ``build_reference_kv`` turns it into the reference K/V the denoising stage hands to +every ``forward`` of that clip. +""" + +import msgspec +import torch + + +class WanAnimate2ClipConditioning(msgspec.Struct, frozen=True, kw_only=True): + """Per-clip conditioning consumed by the DiT's forward_ref / forward_gen.""" + + # Inputs for ``forward_ref`` (reference video = the motion source) + # VAE latents of this clip's reference-video frames, + # [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32. + reference_video_latents: torch.Tensor + # Generation token grid (latent_t, latent_h // 2, latent_w // 2): the DiT patchifies the + # latent with patch_size (1, 2, 2), so each spatial side halves. + generation_video_grid_sizes: tuple[int, int, int] + # CLIP tokens of the clip's first reference-video frame, [1, 257, 1280] bf16. + reference_video_frame_0_image_embeddings: torch.Tensor + # [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16. + # Channels: [0:4] given-frame mask, all ones (every reference frame is known); + # [4:20] VAE latent, the same tensor as reference_video_latents. + # Time: the clip's reference-video latent frames; no reference-image frame in front. + # forward_ref cats it under reference_video_latents (16 ch) for the 36-channel DiT input. + reference_video_condition: torch.Tensor + # T5 embedding of prompt_ref, [L_prompt_ref, 4096] bf16. + prompt_ref_embeddings: torch.Tensor + + # Inputs for: ``forward_gen`` + # [20, latent_t, latent_h, latent_w] bf16. + # Channels: [0:4] given-frame mask (1 = known pixels, 0 = to generate); + # [4:20] VAE latent of the given pixels, encoded zeros elsewhere. + # Time: frame 0 is the reference image (mask = all ones); frames 1..latent_t are the clip's + # own clip_latent_t frames, given only for the num_frames_conditioning frames fed + # back from the previous clip (none for clip 0). + # forward_gen cats it under the noise latent (16 ch) for the 36-channel DiT input. + generation_condition: torch.Tensor + # Reference-video token grid (reference_video_latent_t, _h // 2, _w // 2); same halving. + reference_video_grid_sizes: tuple[int, int, int] + # Token grid of a full-length clip (the configured clip_len); the attention layout and + # block mask are sized from it even on the shorter last clip, so one compiled mask + # serves the whole request. + full_clip_grid_sizes: tuple[int, int, int] + + # Pixel-frame count of this clip (the last clip may be shorter than clip_len). + num_frames: int + # Seeded initial noise for the denoising loop, [16, latent_t, latent_h, latent_w] fp32. + init_noise: torch.Tensor + + +class WanAnimate2ReferenceKV(msgspec.Struct, frozen=True): + """One clip's post-RoPE reference-video K and V, keyed by DiT block index. + + Returned by ``WanAnimate2Transformer3DModel.build_reference_kv`` and owned by the caller + for that clip only; the DiT keeps no copy between calls. + """ + + # [1, S_reference_video, local_heads, head_dim] in the DiT param dtype. Under Ulysses SP + # this is the full sequence with sharded heads, the layout forward_gen uses after its a2a. + k: dict[int, torch.Tensor] + v: dict[int, torch.Tensor] diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py index 66e40dc2c8..01ea542dd7 100755 --- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py @@ -617,6 +617,61 @@ class WanTransformerBlock(nn.Module): self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + def _qkv( + self, norm_hidden_states: torch.Tensor, local_heads: int + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + query, _ = self.to_q(norm_hidden_states) + key, _ = self.to_k(norm_hidden_states) + value, _ = self.to_v(norm_hidden_states) + if self.norm_q is not None: + query = ( + tensor_parallel_rms_norm(query, self.norm_q) + if self.tp_rmsnorm + else self.norm_q(query) + ) + if self.norm_k is not None: + key = ( + tensor_parallel_rms_norm(key, self.norm_k) + if self.tp_rmsnorm + else self.norm_k(key) + ) + return tuple( + tensor.squeeze(1).unflatten(2, (local_heads, self.dim_head)) + for tensor in (query, key, value) + ) + + def _cross_and_ffn( + self, + hidden_states: torch.Tensor, + attn_output: torch.Tensor, + encoder_hidden_states: torch.Tensor, + gate_msa: torch.Tensor, + c_shift_msa: torch.Tensor, + c_scale_msa: torch.Tensor, + c_gate_msa: torch.Tensor, + orig_dtype: torch.dtype, + ) -> torch.Tensor: + attn_output, _ = self.to_out(attn_output) + attn_output = attn_output.squeeze(1) + null_shift = null_scale = torch.zeros( + (1,), device=hidden_states.device, dtype=hidden_states.dtype + ) + norm_hidden_states, hidden_states = self.self_attn_residual_norm( + hidden_states, attn_output, gate_msa, null_shift, null_scale + ) + norm_hidden_states = norm_hidden_states.to(orig_dtype) + hidden_states = hidden_states.to(orig_dtype) + attn_output = self.attn2( + norm_hidden_states, context=encoder_hidden_states, context_lens=None + ) + norm_hidden_states, hidden_states = self.cross_attn_residual_norm( + hidden_states, attn_output, 1, c_shift_msa, c_scale_msa + ) + norm_hidden_states = norm_hidden_states.to(orig_dtype) + hidden_states = hidden_states.to(orig_dtype) + ff_output = self.ffn(norm_hidden_states) + return self.mlp_residual(ff_output, c_gate_msa, hidden_states).to(orig_dtype) + def forward( self, hidden_states: torch.Tensor, @@ -655,23 +710,7 @@ class WanTransformerBlock(nn.Module): # 1. Self-attention norm_hidden_states = self.norm1(hidden_states, shift_msa, scale_msa) - query, _ = self.to_q(norm_hidden_states) - key, _ = self.to_k(norm_hidden_states) - value, _ = self.to_v(norm_hidden_states) - - if self.norm_q is not None: - if self.tp_rmsnorm: - query = tensor_parallel_rms_norm(query, self.norm_q) - else: - query = self.norm_q(query) - if self.norm_k is not None: - if self.tp_rmsnorm: - key = tensor_parallel_rms_norm(key, self.norm_k) - else: - key = self.norm_k(key) - query = query.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head)) - key = key.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head)) - value = value.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head)) + query, key, value = self._qkv(norm_hidden_states, self.local_num_heads) # Apply rotary embeddings cos, sin = freqs_cis @@ -729,38 +768,16 @@ class WanTransformerBlock(nn.Module): attn_output = self.attn1(query, key, value) attn_output = attn_output.flatten(2) - attn_output, _ = self.to_out(attn_output) - attn_output = attn_output.squeeze(1) - - null_shift = null_scale = torch.zeros( - (1,), device=hidden_states.device, dtype=hidden_states.dtype + return self._cross_and_ffn( + hidden_states, + attn_output, + encoder_hidden_states, + gate_msa, + c_shift_msa, + c_scale_msa, + c_gate_msa, + orig_dtype, ) - norm_hidden_states, hidden_states = self.self_attn_residual_norm( - hidden_states, attn_output, gate_msa, null_shift, null_scale - ) - norm_hidden_states, hidden_states = ( - norm_hidden_states.to(orig_dtype), - hidden_states.to(orig_dtype), - ) - - # 2. Cross-attention - attn_output = self.attn2( - norm_hidden_states, context=encoder_hidden_states, context_lens=None - ) - norm_hidden_states, hidden_states = self.cross_attn_residual_norm( - hidden_states, attn_output, 1, c_shift_msa, c_scale_msa - ) - norm_hidden_states, hidden_states = ( - norm_hidden_states.to(orig_dtype), - hidden_states.to(orig_dtype), - ) - - # 3. Feed-forward - ff_output = self.ffn(norm_hidden_states) - hidden_states = self.mlp_residual(ff_output, c_gate_msa, hidden_states) - hidden_states = hidden_states.to(orig_dtype) - - return hidden_states class WanTransformerBlock_VSA(nn.Module): @@ -1057,29 +1074,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): ) # 3. Transformer blocks - attn_backend = get_global_server_args().attention_backend - transformer_block = ( - WanTransformerBlock_VSA - if (attn_backend and attn_backend.lower() == "video_sparse_attn") - else WanTransformerBlock - ) self.blocks = nn.ModuleList( [ - transformer_block( - inner_dim, - config.ffn_dim, - config.num_attention_heads, - config.qk_norm, - config.cross_attn_norm, - config.eps, - config.added_kv_proj_dim, - self._supported_attention_backends - | {AttentionBackendEnum.VIDEO_SPARSE_ATTN}, - prefix=f"blocks.{i}", - attention_type=config.attention_type, - sla_topk=config.sla_topk, - quant_config=quant_config, - ) + self._make_block(index=i, config=config, quant_config=quant_config) for i in range(config.num_layers) ] ) @@ -1127,6 +1124,36 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): self.layer_names = ["blocks"] + def _make_block( + self, + *, + index: int, + config: WanVideoConfig, + quant_config: QuantizationConfig | None, + ) -> nn.Module: + """Block factory; subclasses override it to substitute their block class.""" + attn_backend = get_global_server_args().attention_backend + transformer_block = ( + WanTransformerBlock_VSA + if (attn_backend and attn_backend.lower() == "video_sparse_attn") + else WanTransformerBlock + ) + return transformer_block( + config.num_attention_heads * config.attention_head_dim, + config.ffn_dim, + config.num_attention_heads, + config.qk_norm, + config.cross_attn_norm, + config.eps, + config.added_kv_proj_dim, + self._supported_attention_backends + | {AttentionBackendEnum.VIDEO_SPARSE_ATTN}, + prefix=f"blocks.{index}", + attention_type=config.attention_type, + sla_topk=config.sla_topk, + quant_config=quant_config, + ) + @lru_cache(maxsize=1) def _compute_rope_for_sequence_shard( self, @@ -1338,30 +1365,37 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin): if seq_shard_pad > 0: hidden_states = hidden_states[:, :seq_len_orig, :] - # 5. Output norm, projection & unpatchify - if temb.dim() == 3: + return self._unpatchify( + hidden_states, + temb, + (post_patch_num_frames, post_patch_height, post_patch_width), + ) + + def _unpatchify( + self, + hidden_states: torch.Tensor, + timestep_embeddings: torch.Tensor, + grid_size: tuple[int, int, int], + ) -> torch.Tensor: + """Output norm, projection and token-grid reconstruction for Wan variants.""" + if timestep_embeddings.dim() == 3: # batch_size, seq_len, inner_dim (wan 2.2 ti2v) shift, scale = ( - self.scale_shift_table.unsqueeze(0) + temb.unsqueeze(2) + self.scale_shift_table.unsqueeze(0) + timestep_embeddings.unsqueeze(2) ).chunk(2, dim=2) shift = shift.squeeze(2) scale = scale.squeeze(2) else: # batch_size, inner_dim - shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk(2, dim=1) + shift, scale = ( + self.scale_shift_table + timestep_embeddings.unsqueeze(1) + ).chunk(2, dim=1) hidden_states = self.norm_out(hidden_states, shift, scale) hidden_states, _ = self.proj_out(hidden_states) hidden_states = hidden_states.reshape( - batch_size, - post_patch_num_frames, - post_patch_height, - post_patch_width, - p_t, - p_h, - p_w, - -1, + hidden_states.shape[0], *grid_size, *self.patch_size, -1 ) hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6) output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py index 03c695909b..a6340c7d07 100644 --- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py +++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py @@ -9,6 +9,7 @@ # This wrapper delegates all numerical work to diffusers' implementation # and only adapts the interface for sglang's denoising stage. +import numpy as np import torch from diffusers import ( DPMSolverMultistepScheduler as DiffusersDPMSolverMultistepScheduler, @@ -101,10 +102,43 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler): def begin_index(self) -> int | None: return self._begin_index - def set_timesteps(self, num_inference_steps: int, device=None, **kwargs): - self._inner.set_timesteps(num_inference_steps, device=device, **kwargs) + def set_timesteps( + self, + num_inference_steps: int | None = None, + device=None, + sigmas: np.ndarray | list[float] | None = None, + **kwargs, + ): + if sigmas is None: + self._inner.set_timesteps(num_inference_steps, device=device, **kwargs) + else: + self._set_sigmas(np.asarray(sigmas, dtype=np.float64), device) self.timesteps = self._inner.timesteps + def _set_sigmas(self, sigmas: np.ndarray, device=None) -> None: + inner = self._inner + if inner.config.final_sigmas_type == "sigma_min": + sigma_last = ( + (1 - inner.alphas_cumprod[0]) / inner.alphas_cumprod[0] + ) ** 0.5 + else: + sigma_last = 0 + timesteps = sigmas * inner.config.num_train_timesteps + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + inner.sigmas = torch.from_numpy(sigmas) + inner.timesteps = torch.from_numpy(timesteps).to( + device=device, dtype=torch.int64 + ) + inner.num_inference_steps = len(timesteps) + self._reset_solver_state() + + def _reset_solver_state(self) -> None: + inner = self._inner + inner.model_outputs = [None] * inner.config.solver_order + inner.lower_order_nums = 0 + inner._step_index = None + inner._begin_index = None + def scale_model_input( self, sample: torch.Tensor, timestep: int | None = None ) -> torch.Tensor: diff --git a/python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py new file mode 100644 index 0000000000..e2df14e238 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Wan-Animate-2 pipeline. + +Stages: WanAnimate2BeforeDenoisingStage (per-clip conditioning) -> WanAnimate2DenoisingStage +(per-clip denoise + in-loop VAE decode) -> WanAnimate2OutputStage (emit frames). +Routed by model_index.json ``_class_name == "WanAnimate2Pipeline"``; config comes from +the registry detector (Wan_Animate_2_14B_Config). Every component loads through the +standard component loaders. Text uses TextEncodingStage; CLIP and VAE adapters +preserve the reference preprocessing and latent scaling. +""" + +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed import get_local_torch_device +from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import ( + TransformerLoader, +) +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2 import ( + WanAnimate2BeforeDenoisingStage, + WanAnimate2DenoisingStage, + WanAnimate2ImageEncoderAdapter, + WanAnimate2OutputStage, + WanAnimate2VaeAdapter, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + + +class WanAnimate2TransformerLoader(TransformerLoader): + """The checkpoint is the plain Wan2.2-I2V-14B backbone, so every key must map.""" + + component_names = [] + strict_checkpoint_keys = True + + +class WanAnimate2Pipeline(ComposedPipelineBase): + """Wan-Animate-2 14B in-context reference image + reference video -> video pipeline.""" + + pipeline_name = "WanAnimate2Pipeline" + is_video_pipeline = True + + # All seven model_index.json components. The scheduler is built from the + # checkpoint's scheduler/scheduler_config.json (DPM-Solver++, flow prediction, + # flow sigmas) by the shared loader. Two of its settings are overridden at + # runtime: the stages call set_timesteps(sigmas=...) with the official Wan + # sigma grid at every clip (sigma_0 is exactly 1.0), shifted by + # pipeline_config.flow_shift, so the JSON's flow_shift and timestep_spacing + # are not consulted. The JSON's beta_schedule only shapes the beta-derived + # tables, which the flow-sigma path does not read. + _required_config_modules: list[str] = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + "image_encoder", + "image_processor", + "scheduler", + ] + component_loaders = {"transformer": WanAnimate2TransformerLoader} + + def create_pipeline_stages(self, server_args: ServerArgs) -> None: + # Keep native modules visible to residency and layerwise-offload management; + # adapters only translate the model-specific stage API. + device = get_local_torch_device() + image_encoder = WanAnimate2ImageEncoderAdapter.from_loaded( + self.get_module("image_encoder"), + image_processor=self.get_module("image_processor"), + device=device, + ) + vae = WanAnimate2VaeAdapter(self.get_module("vae")) + self.add_stage( + stage_name="wan_animate_2_before_denoising", + stage=WanAnimate2BeforeDenoisingStage( + vae=vae, + image_encoder=image_encoder, + text_encoder=self.get_module("text_encoder"), + tokenizer=self.get_module("tokenizer"), + pipeline_config=server_args.pipeline_config, + scheduler=self.get_module("scheduler"), + ), + ) + # Decode each clip before constructing the following clip's conditions. + self.add_stage( + stage_name="wan_animate_2_denoising", + stage=WanAnimate2DenoisingStage( + transformer=self.get_module("transformer"), + scheduler=self.get_module("scheduler"), + pipeline=self, + vae=vae, + image_encoder=image_encoder, + ), + ) + # Replaces the standard DecodingStage: frames are already decoded in the loop. + self.add_stage_factory( + RoleType.DECODER, + lambda: WanAnimate2OutputStage(), + "decoding_stage", + ) + + +EntryClass = WanAnimate2Pipeline diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py index 8e9cc4987a..f8ffc1b0bb 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py @@ -234,8 +234,31 @@ class DecodingStage(PipelineStage): Returns: Decoded video tensor with shape (batch, channels, frames, height, width), - normalized to [0, 1] range and moved to CPU as float32 + normalized to [0, 1] on the decode device. """ + normalized_on_frames = ( + (lambda frames: on_frames(_to_unit_range(frames))) + if on_frames is not None + else None + ) + image = self.decode_raw( + latents, + server_args, + vae_dtype=vae_dtype, + on_frames=normalized_on_frames, + ) + return _to_unit_range(image) + + @torch.no_grad() + def decode_raw( + self, + latents: torch.Tensor, + server_args: ServerArgs, + *, + vae_dtype: torch.dtype, + on_frames=None, + ) -> torch.Tensor: + """Decode without pixel normalization, including for inter-clip conditioning.""" latents = latents.to(get_local_torch_device()) # The caller resolves the decode-only override before component use so # residency and execution agree on the target dtype. @@ -270,9 +293,7 @@ class DecodingStage(PipelineStage): latents = latents.to(vae_dtype) decode_kwargs = {} if on_frames is not None: - decode_kwargs["on_frames"] = lambda frames: on_frames( - _to_unit_range(frames) - ) + decode_kwargs["on_frames"] = on_frames with temporary_module_dtype( self.vae, vae_dtype, enabled=should_cast_vae ) as vae: @@ -304,8 +325,6 @@ class DecodingStage(PipelineStage): raise image = _ensure_tensor_decode_output(decode_output) - # De-normalize image to [0, 1] range - image = _to_unit_range(image) return image def load_model(self): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py index 7de0fadc7d..7012258b9f 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py @@ -2084,11 +2084,40 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): neg_cond_kwargs=ctx.neg_cond_kwargs, guidance=ctx.guidance, ) + timesteps_cpu = self._run_denoising_loop(ctx, batch, server_args) + + # collect rollout outputs before finalization gathers or replaces latents + if batch.rollout: + self._postprocess_rollout_outputs( + batch=batch, + latents=ctx.latents, + num_inference_steps=len(timesteps_cpu), + final_timestep=timesteps_cpu.new_zeros(()), + server_args=server_args, + ) + self._finalize_denoising_loop(ctx, batch, server_args) + return batch + + def _run_denoising_loop( + self, + ctx: DenoisingContext, + batch: Req, + server_args: ServerArgs, + *, + collect_trajectory: bool = True, + ) -> torch.Tensor: + """Run a prepared loop, including profiling and DiT residency. + + Clip-based models can reuse this without finalizing the entire request. + """ denoising_start_time = time.time() self._before_denoising_loop(ctx, batch, server_args) # to avoid device-sync caused by timestep comparison timesteps_cpu = ctx.timesteps.cpu() num_timesteps = timesteps_cpu.shape[0] + progress_step_interval = ctx.extra.get( + "progress_step_interval", ctx.scheduler.order + ) # Re-resolve the explicit-range gate so the per-step markers # below honor this request's is_warmup state. Layer hooks are # registered by the residency manager at the use-site. @@ -2132,7 +2161,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): # BEFORE _run_denoising_step so ctx.latents is still the # pre-step value. Gated on batch.rollout to keep the # non-rollout path strictly untouched. - if batch.rollout: + if collect_trajectory and batch.rollout: batch._rollout_loop_step_index = step_index self._maybe_append_dit_trajectory_step( batch=batch, @@ -2141,11 +2170,12 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): step_index=step_index, ) self._run_denoising_step(ctx, step, batch, server_args) - self._record_trajectory(ctx, step, batch, server_args) + if collect_trajectory: + self._record_trajectory(ctx, step, batch, server_args) if step_index == num_timesteps - 1 or ( (step_index + 1) > ctx.num_warmup_steps - and (step_index + 1) % ctx.scheduler.order == 0 + and (step_index + 1) % progress_step_interval == 0 and progress_bar is not None ): progress_bar.update() @@ -2165,20 +2195,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin): del step self._finish_active_component_use() - # Rollout postprocessing must run BEFORE _finalize_denoising_loop so - # the final scheduler.step output (ctx.latents) is still SP-sharded and - # can be gathered uniformly alongside the per-step dit_trajectory via - # gather_stacked_latents_for_sp. - if batch.rollout: - self._postprocess_rollout_outputs( - batch=batch, - latents=ctx.latents, - num_inference_steps=num_timesteps, - final_timestep=timesteps_cpu.new_zeros(()), - server_args=server_args, - ) - self._finalize_denoising_loop(ctx, batch, server_args) - return batch + return timesteps_cpu def _get_extra_func_kwarg_names(self, func) -> tuple[bool, frozenset[str]]: import functools diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py new file mode 100644 index 0000000000..972b2b898b --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Wan-Animate-2-specific pipeline stages.""" + +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import ( + WanAnimate2BeforeDenoisingStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import ( + WanAnimate2DenoisingStage, + WanAnimate2OutputStage, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import ( + WanAnimate2ImageEncoderAdapter, + WanAnimate2VaeAdapter, +) + +__all__ = [ + "WanAnimate2BeforeDenoisingStage", + "WanAnimate2DenoisingStage", + "WanAnimate2OutputStage", + "WanAnimate2ImageEncoderAdapter", + "WanAnimate2VaeAdapter", +] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py new file mode 100644 index 0000000000..5185833642 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Reference-video audio extraction for the Wan-Animate-2 before-denoising stage.""" + +from __future__ import annotations + +from collections.abc import Callable + +import torch + +from sglang.srt.multimodal.audio_from_video import decode_audio_container + +# Logger callables with the ``PipelineStage.log_*`` signature: ``(msg, *args)``. +LogFn = Callable[..., None] + + +def extract_reference_video_audio( + reference_video_path: str, + *, + log_info: LogFn, + log_warning: LogFn, + log_no_audio_track: LogFn | None = None, +) -> tuple[torch.Tensor | None, int | None]: + """Return ``(audio, sample_rate)`` for ``OutputBatch.audio``: audio ``[1, C, L]`` fp32 in + [-1, 1] (dim 0 is the per-output axis the save path indexes) at the track's native sample + rate and channel count, or ``(None, None)`` when the video has no decodable audio track. + The track is decoded in-process by the shared ``decode_audio_container`` reader (PyAV). + Best-effort: every failure is logged, none is raised. A video without an audio track goes + to ``log_no_audio_track`` (``log_warning`` when None); a track that cannot be decoded goes + to ``log_warning`` with the reader's message.""" + try: + sample_rate = _audio_track_sample_rate(reference_video_path) + if sample_rate is None: + (log_no_audio_track or log_warning)( + "Wan-Animate-2: reference video %s has no audio track; the output is silent.", + reference_video_path, + ) + return None, None + # [L, C] fp32 in [-1, 1]; the reader needs a target rate, so the native one keeps the + # track unresampled. + samples = decode_audio_container( + reference_video_path, target_sr=sample_rate, mono=False + ) + except Exception as e: # audio is best-effort: a failure never fails the request + log_warning( + "Wan-Animate-2: cannot decode the audio track of reference video %s (%s); " + "the output is silent.", + reference_video_path, + e, + ) + return None, None + + audio = torch.from_numpy(samples).T.contiguous().unsqueeze(0) # [1, C, L] + log_info( + "Wan-Animate-2: extracted reference-video audio %s @ %d Hz.", + tuple(audio.shape), + sample_rate, + ) + return audio, sample_rate + + +def _audio_track_sample_rate(path: str) -> int | None: + """Native sample rate of the video's first audio stream; None when it has none.""" + import av + + with av.open(path) as container: + if not container.streams.audio: + return None + return int(container.streams.audio[0].rate) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py new file mode 100644 index 0000000000..b36b00cbb6 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py @@ -0,0 +1,706 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Wan-Animate-2 before-denoising stage. + +Builds the per-clip conditioning container (``WanAnimate2ClipConditioning``) the DiT +reads via its ``clip_cond`` kwarg, plus the clip-invariant text/CLIP/VAE encodings. +Preprocessing helpers live in ``preprocess.py`` next to this module. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +import cv2 +import msgspec +import numpy as np +import torch +from einops import rearrange + +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + get_sp_world_size, +) +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import ( + WanAnimate2ClipConditioning, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import ( + DPMSolverMultistepScheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio import ( + extract_reference_video_audio, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import ( + LetterboxInfo, + get_padding_len, + make_conditioning_mask, + read_reference_video_frames, + resize_by_area, + validate_and_get_single_string, + zigzag_padding, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs + +if TYPE_CHECKING: + from sglang.multimodal_gen.configs.pipeline_configs.wan import ( + Wan_Animate_2_14B_Config, + ) + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import ( + WanAnimate2ImageEncoderAdapter, + WanAnimate2VaeAdapter, + ) + +# Frames of the previous clip's decoded output that open the next clip as given (not denoised); +# the official pipelines/wan_animate_2_pipeline.py hard-codes num_frames_conditioning = 1. +_NUM_FRAMES_CONDITIONING = 1 +# Wan VAE temporal compression: 4 pixel frames per latent frame after the first. +_VAE_TEMPORAL_COMPRESSION = 4 +# Wan VAE latent channels. +_NUM_LATENT_CHANNELS = 16 + + +def generation_token_grid( + num_frames: int, latent_h: int, latent_w: int +) -> tuple[int, int, int]: + """DiT token grid (f, h, w) of a clip of ``num_frames`` pixel frames. Wan VAE: 4k + 1 pixel + frames -> k + 1 latent frames; one latent frame is prepended for the reference image; + the DiT patchifies with patch_size (1, 2, 2), so each spatial side halves.""" + clip_latent_t = (num_frames - 1) // _VAE_TEMPORAL_COMPRESSION + 1 + return clip_latent_t + 1, latent_h // 2, latent_w // 2 + + +def _generation_latent_shape( + latent_h: int, latent_w: int, num_frames: int +) -> tuple[int, int, int, int]: + """``[16, latent_t, latent_h, latent_w]`` of the latent the DiT denoises for a clip of + ``num_frames`` pixel frames; latent_t comes from generation_token_grid (clip frames plus + the leading reference-image slot).""" + latent_t = generation_token_grid(num_frames, latent_h, latent_w)[0] + return (_NUM_LATENT_CHANNELS, latent_t, latent_h, latent_w) + + +def get_sampling_sigmas(sampling_steps: int, shift: float) -> np.ndarray: + """Official Wan sampler grid: ``[sampling_steps]`` fp64 sigmas, uniform on [1, 0) with ``shift`` applied (sigma_0 == 1).""" + sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps] + sigma = shift * sigma / (1 + (shift - 1) * sigma) + return sigma + + +class _PerClipDenoisingMetadata(msgspec.Struct, frozen=True, kw_only=True): + """Schedule metadata for one clip of the reference video.""" + + # Position in the schedule, 0-based. + clip_index: int + # Start frame (inclusive) in the padded reference video. + frame_start_index: int + # End frame (exclusive) in the padded reference video. + frame_end_index: int + # Pixel-frame count, frame_end_index - frame_start_index, zigzag-padded frames included; + # shorter than clip_len only on the last clip. + num_frames: int + # Leading frames taken from the previous clip's output: 0 for clip 0, else + # num_frames_conditioning. + num_frames_conditioning_for_clip: int + + +def build_schedule( + reference_video_frames: list[np.ndarray], + clip_len: int, + num_frames_conditioning: int, +) -> list[_PerClipDenoisingMetadata]: + """Prepare denoising metadata for all the clips of the reference video.""" + schedule: list[_PerClipDenoisingMetadata] = [] + start = 0 + clip_index = 0 + num_frames_reference_video = len(reference_video_frames) + while True: + if start + num_frames_conditioning >= num_frames_reference_video: + break + num_frames_conditioning_for_clip = 0 if start == 0 else num_frames_conditioning + clip_len_current = clip_len + if num_frames_reference_video - start < clip_len: # last clip shrinks + clip_len_current = num_frames_reference_video - start + schedule.append( + _PerClipDenoisingMetadata( + clip_index=clip_index, + frame_start_index=start, + frame_end_index=start + clip_len_current, + num_frames=clip_len_current, + num_frames_conditioning_for_clip=num_frames_conditioning_for_clip, + ) + ) + start += clip_len_current - num_frames_conditioning + clip_index += 1 + if not schedule: + raise ValueError( + "Wan-Animate-2 reference video produced 0 clips; check " + "clip_len/num_frames_conditioning and that the video decoded frames." + ) + return schedule + + +class _WanAnimate2Inputs(msgspec.Struct, frozen=True, kw_only=True): + """The per-request inputs read off the Req.""" + + reference_image_path: str + reference_video_path: str + + prompt: str + prompt_ref: str + negative_prompt: str + + width: int + height: int + clip_len: int + num_frames_conditioning: int + fps: int + seed: int + num_inference_steps: int + guidance_scale: float + enable_audio: bool + + @classmethod + def from_request(cls, batch: Req) -> _WanAnimate2Inputs: + """Read the Wan-Animate-2 input fields off the Req (all SamplingParam-backed).""" + reference_image_path = validate_and_get_single_string( + batch.image_path, "image_path" + ) + reference_video_path = validate_and_get_single_string( + batch.video_path, "video_path" + ) + prompt = validate_and_get_single_string(batch.prompt, "prompt") + prompt_ref = validate_and_get_single_string(batch.prompt_ref, "prompt_ref") + # negative_prompt_embeddings is always computed; an unset negative prompt encodes "". + negative_prompt = validate_and_get_single_string( + batch.negative_prompt or "", "negative_prompt" + ) + + seed = batch.seed + if isinstance(seed, list) and len(seed) != 1: + raise ValueError( + f"Wan-Animate-2 takes exactly one 'seed' per request, got {len(seed)}" + ) + if isinstance(seed, list): + seed = seed[0] + + return _WanAnimate2Inputs( + reference_image_path=reference_image_path, + reference_video_path=reference_video_path, + prompt=prompt, + prompt_ref=prompt_ref, + negative_prompt=negative_prompt, + width=int(batch.width), + height=int(batch.height), + clip_len=int(batch.clip_len), + num_frames_conditioning=_NUM_FRAMES_CONDITIONING, + fps=int(batch.fps), + seed=int(seed), + num_inference_steps=int(batch.num_inference_steps), + guidance_scale=float(batch.guidance_scale), + enable_audio=bool(batch.enable_audio), + ) + + +class WanAnimate2RequestState(msgspec.Struct, kw_only=True): + """Clip-independent state, computed once, reused by every clip.""" + + device: torch.device + sp_size: int + generator: torch.Generator + inputs: _WanAnimate2Inputs + + # reference image + letterbox_info: LetterboxInfo + reference_image_height: int + reference_image_width: int + latent_h: int # reference_image_height // 8 + latent_w: int # reference_image_width // 8 + reference_image_condition: torch.Tensor # [20, 1, latent_h, latent_w] bf16 + reference_image_embeddings: torch.Tensor # [1, 257, 1280] bf16 + + # text + prompt_embeddings: torch.Tensor # [L, 4096] bf16 + negative_prompt_embeddings: torch.Tensor # [L, 4096] bf16 + prompt_ref_embeddings: torch.Tensor # [L_prompt_ref, 4096] bf16 + + # reference video, zigzag-padded to the clip schedule + reference_video_frames: list[np.ndarray] # resized [H, W, 3] uint8 RGB frames + schedule: list[_PerClipDenoisingMetadata] + # frame count before padding; the final video is trimmed to it + num_reference_video_frames: int + + # Token grid of a full-length clip; every clip's attention layout is padded to it. + full_clip_grid_sizes: tuple[int, int, int] + + # Reference-video audio, opt-in per request via enable_audio; None gives a silent mp4. + audio: torch.Tensor | None = None # [1, C, L] fp32 in [-1, 1] + audio_sample_rate: int | None = None + # Assembled by the denoising stage for the output stage, [T, H, W, C] uint8. + decoded_frames: np.ndarray | None = None + # Initial noise for clip 0, drawn by the before-denoising stage so that batch.latents is + # the real denoising input; later clips draw from the generator in schedule order. + clip_0_init_noise: torch.Tensor | None = None + + +# The one batch.extra entry Wan-Animate-2 uses; later stages read it back through +# request_state_from_batch. +WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY = "wan_animate_2_request_state" + + +def request_state_from_batch(batch: Req) -> WanAnimate2RequestState: + state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY) + if not isinstance(state, WanAnimate2RequestState): + raise ValueError( + f"batch.extra[{WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY!r}] is missing: " + "WanAnimate2BeforeDenoisingStage must run before this stage." + ) + return state + + +def _mirror_request_state_into_batch( + batch: Req, + request_state: WanAnimate2RequestState, + *, + scheduler: DPMSolverMultistepScheduler, + flow_shift: float, +) -> None: + """Populate the standard DenoisingStage input fields from the request state. + + The clip loop keeps reading ``request_state``; the embeddings, generator, sigma grid and + timesteps exist for the shared stage verification, warmup and residency bookkeeping. + ``batch.latents`` is clip 0's initial noise, drawn here (the seed's first draw) and + consumed by ``build_clip_conditioning`` for clip 0, so the sequence of generator draws is + the same as when the loop drew it: numerics are unchanged.""" + inputs = request_state.inputs + batch.prompt_embeds = [request_state.prompt_embeddings] + batch.negative_prompt_embeds = [request_state.negative_prompt_embeddings] + batch.image_embeds = [request_state.reference_image_embeddings] + batch.generator = request_state.generator + + sigmas = get_sampling_sigmas(inputs.num_inference_steps, flow_shift) + scheduler.set_timesteps(sigmas=sigmas, device=request_state.device) + batch.timesteps = scheduler.timesteps + batch.sigmas = sigmas.tolist() + + batch.latents = _draw_clip_0_init_noise(request_state) + batch.raw_latent_shape = batch.latents.shape + + +def _draw_clip_0_init_noise(request_state: WanAnimate2RequestState) -> torch.Tensor: + """Draw clip 0's initial noise from the request generator (its first draw) and keep it on + the request state for build_clip_conditioning; clips 1.. then take the following draws.""" + init_noise = torch.randn( + *_generation_latent_shape( + request_state.latent_h, + request_state.latent_w, + request_state.schedule[0].num_frames, + ), + dtype=torch.float32, + device=request_state.device, + generator=request_state.generator, + ) + request_state.clip_0_init_noise = init_noise + return init_noise + + +VaeEncoderFn = Callable[[list[torch.Tensor]], list[torch.Tensor]] +ImageEmbedderFn = Callable[[list[torch.Tensor]], torch.Tensor] + + +def build_clip_conditioning( + request_state: WanAnimate2RequestState, + clip_denoising_metadata: _PerClipDenoisingMetadata, + prev_clip_conditioning_frames: torch.Tensor | None, + *, + vae_encoder: VaeEncoderFn, + image_embedder: ImageEmbedderFn, +) -> WanAnimate2ClipConditioning: + """Build conditioning for a single clip.""" + device = request_state.device + latent_h, latent_w = request_state.latent_h, request_state.latent_w + reference_image_height, reference_image_width = ( + request_state.reference_image_height, + request_state.reference_image_width, + ) + + # gen/noise latent geometry + num_frames = clip_denoising_metadata.num_frames + grid_sizes = generation_token_grid(num_frames, latent_h, latent_w) + latent_t = grid_sizes[0] + # The clip's own latent frames, without the prepended reference-image frame. + clip_latent_t = latent_t - 1 + init_noise_shape = _generation_latent_shape(latent_h, latent_w, num_frames) + + with ( + torch.autocast(device_type=device.type, dtype=torch.bfloat16), + torch.no_grad(), + ): + # previous-clip condition over the clip's own latent frames (no reference-image frame) + # [4, clip_latent_t, latent_h, latent_w] fp32 + prev_clip_condition_mask = make_conditioning_mask( + clip_latent_t, + latent_h, + latent_w, + clip_denoising_metadata.num_frames_conditioning_for_clip, + device=device, + ) + # The clip's pixel video: the given frames first, zeros where the DiT generates. + # Built on ``device`` so the frames never leave the GPU. + prev_clip_condition_pixel_space = torch.zeros( + 3, + num_frames, + reference_image_height, + reference_image_width, + device=device, + ) + if clip_denoising_metadata.num_frames_conditioning_for_clip > 0: + # prev_clip_conditioning_frames: the previous clip's last + # ``num_frames_conditioning`` decoded frames, + # [C, num_frames_conditioning, reference_image_height, reference_image_width] bf16 in [-1, 1]. + if prev_clip_conditioning_frames is None: + raise ValueError( + "For all clips except clip-0 `prev_clip_conditioning_frames` must be not None." + ) + + prev_clip_condition_pixel_space[ + :, : clip_denoising_metadata.num_frames_conditioning_for_clip + ] = torch.nn.functional.interpolate( + prev_clip_conditioning_frames[ + :, : clip_denoising_metadata.num_frames_conditioning_for_clip + ], + size=(reference_image_height, reference_image_width), + mode="bicubic", + ) + + # [16, clip_latent_t, latent_h, latent_w] fp32 + prev_clip_condition = vae_encoder([prev_clip_condition_pixel_space])[0] + + # [20, clip_latent_t, latent_h, latent_w] bf16 + prev_clip_condition = torch.concat( + [prev_clip_condition_mask, prev_clip_condition] + ).to(torch.bfloat16) + + # [20, latent_t, latent_h, latent_w] bf16 + # recall: latent_t = clip_latent_t + 1 (+1 is for the reference image condition in latent space) + generation_condition = torch.concat( + [request_state.reference_image_condition, prev_clip_condition], dim=1 + ) + + # reference-video slice -> reference_video_latents + # [t, h, w, c] uint8 + reference_video_clip_np = np.stack( + request_state.reference_video_frames[ + clip_denoising_metadata.frame_start_index : clip_denoising_metadata.frame_end_index + ] + ) + reference_video_pixels = rearrange( + torch.from_numpy(reference_video_clip_np).to(device), "t h w c -> c t h w" + ) + # [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, t, h, w] + reference_video_pixels = (reference_video_pixels.float() / 127.5 - 1).to( + torch.bfloat16 + ) + + # [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32 + # Stays fp32 (vae.encode returns .float()); forward_ref cats it with the + # bf16 reference_video_condition under autocast type promotion. + reference_video_latents = vae_encoder([reference_video_pixels])[0] + ( + _, + reference_video_latent_t, + reference_video_latent_h, + reference_video_latent_w, + ) = reference_video_latents.shape + reference_video_grid_sizes = ( + reference_video_latent_t, + reference_video_latent_h // 2, + reference_video_latent_w // 2, + ) + + # [3, reference_video_height, reference_video_width] bf16 + reference_video_frame_0 = reference_video_pixels[:, 0] + reference_video_frame_0_image_embeddings = image_embedder( + [reference_video_frame_0[:, None, :, :]] + ).to(torch.bfloat16) # [1, 257, 1280] bf16 + + # reference_video_condition (forward_ref): mask, then VAE latent of the reference-video slice. + # Every frame of the slice is given, so the latent is reference_video_latents itself; + # the official code encodes the same pixels a second time here. + reference_video_condition_mask = make_conditioning_mask( + reference_video_latent_t, + reference_video_latent_h, + reference_video_latent_w, + num_frames, + device=device, + ) + # [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16 + reference_video_condition = torch.concat( + [reference_video_condition_mask, reference_video_latents] + ).to(torch.bfloat16) + + # Drawn from the request's seeded generator; clips are built in schedule order, so + # clip k always gets the k-th draw and (seed, clip_index) identifies the noise. Clip 0's + # draw already happened in the before-denoising stage (it is batch.latents). + if ( + clip_denoising_metadata.clip_index == 0 + and request_state.clip_0_init_noise is not None + ): + init_noise = request_state.clip_0_init_noise + else: + init_noise = torch.randn( + *init_noise_shape, + dtype=torch.float32, + device=device, + generator=request_state.generator, + ) + + return WanAnimate2ClipConditioning( + reference_video_latents=reference_video_latents, + generation_video_grid_sizes=grid_sizes, + reference_video_frame_0_image_embeddings=reference_video_frame_0_image_embeddings, + reference_video_condition=reference_video_condition, + prompt_ref_embeddings=request_state.prompt_ref_embeddings, + generation_condition=generation_condition, + reference_video_grid_sizes=reference_video_grid_sizes, + full_clip_grid_sizes=request_state.full_clip_grid_sizes, + num_frames=clip_denoising_metadata.num_frames, + init_noise=init_noise, + ) + + +class WanAnimate2BeforeDenoisingStage(TextEncodingStage): + """Native text encoding plus the model-specific reference and clip conditioning.""" + + # Text-only stage dedup must not skip fresh image/video conditions or the request RNG. + deduplicated_output_fields = () + + def __init__( + self, + vae: WanAnimate2VaeAdapter, + image_encoder: WanAnimate2ImageEncoderAdapter, + text_encoder: torch.nn.Module, + tokenizer, + pipeline_config: Wan_Animate_2_14B_Config, + scheduler: DPMSolverMultistepScheduler, + ) -> None: + super().__init__(text_encoders=[text_encoder], tokenizers=[tokenizer]) + self.vae = vae + self.image_encoder = image_encoder + self.pipeline_config = pipeline_config + self._silent_reference_logged = False + self._audio_failure_logged = False + # Shared with the denoising stage, which resets it per clip; used here only to derive + # batch.timesteps on the same sigma grid. + self.scheduler = scheduler + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + name = self._component_stage_name(stage_name) + return super().component_uses(server_args, stage_name) + [ + ComponentUse(stage_name=name, component_name="vae"), + ComponentUse(stage_name=name, component_name="image_encoder"), + ] + + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + device = get_local_torch_device() + sp_size = max(1, int(get_sp_world_size())) + + inputs = _WanAnimate2Inputs.from_request(batch) + + # One seeded generator per request; clips draw their noise from it in order. + generator = torch.Generator(device=device).manual_seed(inputs.seed) + + audio, audio_sample_rate = self._extract_audio(inputs) + request_state = self._build_request_state( + inputs, + device, + sp_size, + generator, + server_args, + audio=audio, + audio_sample_rate=audio_sample_rate, + ) + if len(request_state.schedule) > 1: + self.log_info( + "reference video splits into %d clips; each is conditioned in the denoising " + "loop via build_clip_conditioning.", + len(request_state.schedule), + ) + batch.extra[WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY] = request_state + _mirror_request_state_into_batch( + batch, + request_state, + scheduler=self.scheduler, + flow_shift=self.pipeline_config.flow_shift, + ) + return batch + + def _build_request_state( + self, + inputs: _WanAnimate2Inputs, + device: torch.device, + sp_size: int, + generator: torch.Generator, + server_args: ServerArgs, + *, + audio: torch.Tensor | None, + audio_sample_rate: int | None, + ) -> WanAnimate2RequestState: + width, height = inputs.width, inputs.height + + reference_image = cv2.imread(inputs.reference_image_path) + if reference_image is None: + raise ValueError( + f"Wan-Animate-2: failed to read reference image at {inputs.reference_image_path!r} " + "(cv2.imread returned None: missing file or unsupported/corrupt image)." + ) + reference_image = reference_image[..., ::-1] # BGR -> RGB + reference_image, letterbox_info = resize_by_area( + reference_image, width * height, divisor=16 + ) + + # reference video: decode, fps resample, zigzag pad. + frames = read_reference_video_frames(inputs.reference_video_path, inputs.fps) + reference_video_frames = [ + resize_by_area(f, width * height, divisor=16)[0] for f in frames + ] + num_reference_video_frames = len(frames) + target_len = get_padding_len(num_reference_video_frames, inputs.clip_len) + reference_video_frames = zigzag_padding(reference_video_frames, target_len) + + # Keep the official padding and geometry of every needed clip, but do not + # denoise clips whose entire output would be discarded by the final trim. + schedule = [ + clip + for clip in build_schedule( + reference_video_frames, inputs.clip_len, inputs.num_frames_conditioning + ) + if clip.frame_start_index + clip.num_frames_conditioning_for_clip + < num_reference_video_frames + ] + + # reference-image encodings (constant across clips) + reference_image_height, reference_image_width = ( + reference_image.shape[0], + reference_image.shape[1], + ) + latent_h, latent_w = reference_image_height // 8, reference_image_width // 8 + + reference_image_pixel_values = rearrange( + torch.from_numpy(reference_image).to(device), "h w c -> c 1 h w" + ) + # [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, 1, H, W] + reference_image_pixel_values = ( + reference_image_pixel_values.float() / 127.5 - 1 + ).to(torch.bfloat16) + + with ( + torch.autocast(device_type=device.type, dtype=torch.bfloat16), + torch.no_grad(), + ): + # [16, 1, latent_h, latent_w] fp32 + reference_image_latents = self.vae_encoder([reference_image_pixel_values])[ + 0 + ] + + # [4, 1, latent_h, latent_w] fp32 + reference_image_mask = make_conditioning_mask( + 1, latent_h, latent_w, 1, device=device + ) + + # [20, 1, latent_h, latent_w] bf16 + reference_image_condition = torch.concat( + [reference_image_mask, reference_image_latents] + ).to(torch.bfloat16) + + reference_image_embeddings = self.image_embedder( + [reference_image_pixel_values] + ).to(torch.bfloat16) # [1, 257, 1280] bf16 + + # Keep batch-one encoder arithmetic while sharing native caching and residency. + prompt_embeddings, prompt_ref_embeddings, negative_prompt_embeddings = [ + self.encode_text(prompt, server_args, device=device)[0][0] + for prompt in (inputs.prompt, inputs.prompt_ref, inputs.negative_prompt) + ] + + return WanAnimate2RequestState( + device=device, + sp_size=sp_size, + generator=generator, + inputs=inputs, + letterbox_info=letterbox_info, + reference_image_height=reference_image_height, + reference_image_width=reference_image_width, + latent_h=latent_h, + latent_w=latent_w, + reference_image_condition=reference_image_condition, + reference_image_embeddings=reference_image_embeddings, + prompt_embeddings=prompt_embeddings, + negative_prompt_embeddings=negative_prompt_embeddings, + prompt_ref_embeddings=prompt_ref_embeddings, + reference_video_frames=reference_video_frames, + schedule=schedule, + num_reference_video_frames=num_reference_video_frames, + full_clip_grid_sizes=generation_token_grid( + inputs.clip_len, latent_h, latent_w + ), + audio=audio, + audio_sample_rate=audio_sample_rate, + ) + + def _extract_audio( + self, inputs: _WanAnimate2Inputs + ) -> tuple[torch.Tensor | None, int | None]: + """Per-request opt-out; any failure yields (None, None), a silent video.""" + if not inputs.enable_audio: + return None, None + return extract_reference_video_audio( + inputs.reference_video_path, + log_info=self.log_info, + log_no_audio_track=self._log_silent_reference, + log_warning=self._log_audio_failure, + ) + + def _log_silent_reference(self, msg: str, *args: Any) -> None: + # A reference video without an audio track is normal input, not a failure: + # one info line per process, then debug. + if self._silent_reference_logged: + self.log_debug(msg, *args) + return + self._silent_reference_logged = True + self.log_info( + msg + " Set enable_audio=false on the request to skip audio extraction.", + *args, + ) + + def _log_audio_failure(self, msg: str, *args: Any) -> None: + # A track the audio reader cannot decode is the same for every request: warn once + # per process with the remedy, then keep the detail at debug. + if self._audio_failure_logged: + self.log_debug(msg, *args) + return + self._audio_failure_logged = True + self.log_warning( + msg + + " Set enable_audio=false on the request to skip audio extraction; further " + "audio failures are logged at debug level.", + *args, + ) + + def vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]: + with self.use_declared_component(component_name="vae", module=self.vae.vae): + return self.vae.encode(videos) + + def image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor: + with self.use_declared_component( + component_name="image_encoder", module=self.image_encoder.model + ): + return self.image_encoder.visual(videos) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py new file mode 100644 index 0000000000..a816fa71c3 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py @@ -0,0 +1,532 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Wan-Animate-2 denoising and output stages. + +WanAnimate2DenoisingStage orchestrates clips around the shared denoising loop: the in-context DiT +takes per-clip conditioning via its ``clip_cond`` kwarg and the clip's reference K/V via +``reference_kv`` (built once per clip with ``build_reference_kv`` and dropped with the clip), +runs cond/uncond CFG as two forwards (the unconditional branch skips block 9 via +``is_unconditional``), resets the scheduler per clip on the official sigma grid, and +VAE-decodes each clip in-loop to feed the next. +WanAnimate2OutputStage then emits the assembled frames. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING + +import numpy as np +import torch +from einops import rearrange +from torch import nn + +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed import ( + get_local_torch_device, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import ( + run_cfg_parallel, +) +from sglang.multimodal_gen.runtime.distributed.cfg_policy import ( + CFGBranch, + CFGPolicy, +) +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_classifier_free_guidance_world_size, +) +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import ( + WanAnimate2ClipConditioning, + WanAnimate2ReferenceKV, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import ( + DPMSolverMultistepScheduler, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import ( + DenoisingContext, + DenoisingStage, + DenoisingStepState, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import ( + WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY, + WanAnimate2RequestState, + build_clip_conditioning, + get_sampling_sigmas, + request_state_from_batch, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import ( + WanAnimate2ImageEncoderAdapter, + WanAnimate2VaeAdapter, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import ( + LetterboxInfo, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( + V, + VerificationResult, +) +from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger + +logger = init_logger(__name__) + +if TYPE_CHECKING: + from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, + ) + + +def _is_request_state(value: object) -> bool: + return isinstance(value, WanAnimate2RequestState) + + +def _is_decoded_frames(value: object) -> bool: + """``[T, H, W, C]`` uint8 frames assembled by the denoising stage.""" + return isinstance(value, np.ndarray) and value.ndim == 4 and value.dtype == np.uint8 + + +@dataclass(kw_only=True) +class WanAnimate2DenoisingContext(DenoisingContext): + clip_condition: WanAnimate2ClipConditioning + reference_image_embeddings: torch.Tensor + cfg_parallel: bool + guidance_scale: float + reference_kv: WanAnimate2ReferenceKV | None = None + context_by_branch: dict[str, torch.Tensor] = field(default_factory=dict) + + +class WanAnimate2DenoisingStage(DenoisingStage): + """Wan-Animate-2 in-context denoising over all reference-video clips. + + Clip 0 uses the precomputed conditioning; clips > 0 are built from the previous clip's decoded + frames, denoised, decoded in-loop, overlap-trimmed, and assembled into the full video. + """ + + def __init__( + self, + *, + transformer: nn.Module, + scheduler: DPMSolverMultistepScheduler, + pipeline: ComposedPipelineBase, + vae: WanAnimate2VaeAdapter, + image_encoder: WanAnimate2ImageEncoderAdapter, + ) -> None: + super().__init__( + transformer=transformer, scheduler=scheduler, pipeline=pipeline, vae=vae + ) + self.vae: WanAnimate2VaeAdapter = vae + self.clip_decoder = DecodingStage(vae, pipeline=pipeline) + # Clips > 0 are conditioned inside this stage, so it holds the image encoder too. + self.image_encoder = image_encoder + + def _vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]: + with self.use_declared_component(component_name="vae", module=self.vae.vae): + return self.vae.encode(videos) + + def _image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor: + with self.use_declared_component( + component_name="image_encoder", module=self.image_encoder.model + ): + return self.image_encoder.visual(videos) + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.min_dims(1)]) + result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(4)]) + result.add_check("raw_latent_shape", batch.raw_latent_shape, V.not_none) + result.add_check("sigmas", batch.sigmas, V.list_not_empty) + result.add_check( + "prompt_embeds", batch.prompt_embeds, V.list_of_tensors_dims(2) + ) + result.add_check( + "negative_prompt_embeds", + batch.negative_prompt_embeds, + V.list_of_tensors_dims(2), + ) + result.add_check("image_embeds", batch.image_embeds, V.list_of_tensors_dims(3)) + result.add_check( + "num_inference_steps", batch.num_inference_steps, V.positive_int + ) + result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float) + result.add_check("generator", batch.generator, V.generator_or_list_generators) + result.add_check( + WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY, + batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY), + _is_request_state, + ) + return result + + def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY) + result.add_check( + "decoded_frames", + ( + request_state.decoded_frames + if isinstance(request_state, WanAnimate2RequestState) + else None + ), + _is_decoded_frames, + ) + return result + + def component_uses( + self, server_args: ServerArgs, stage_name: str | None = None + ) -> list[ComponentUse]: + # build_clip_conditioning runs CLIP.visual inside this stage, so image_encoder must stay + # resident (it is CPU-offloaded by default for video tasks). + uses = super().component_uses(server_args, stage_name) + # The VAE adapter runs the VAE in fp32 with autocast off; do not let the manager + # cast it to the configured vae precision on the way in. + uses = [ + replace(use, target_dtype=None) if use.component_name == "vae" else use + for use in uses + ] + if not any(use.component_name == "image_encoder" for use in uses): + uses.append( + ComponentUse( + stage_name=self._component_stage_name(stage_name), + component_name="image_encoder", + ) + ) + return uses + + @torch.no_grad() + def _denoise_single_clip( + self, + batch: Req, + server_args: ServerArgs, + request_state: WanAnimate2RequestState, + clip_condition: WanAnimate2ClipConditioning, + ) -> torch.Tensor: + """Denoise one clip; return the latents with the reference-image slot dropped. + ``batch`` is handed to set_forward_context, which the attention backends read, and + supplies the metrics object the per-step records go to.""" + device = get_local_torch_device() + + prompt_embeddings = request_state.prompt_embeddings + negative_prompt_embeddings = request_state.negative_prompt_embeddings + reference_image_embeddings = request_state.reference_image_embeddings + + # Official sampler grid: sigma_0 is exactly 1.0, so sigmas are set explicitly. + # set_timesteps also resets all solver state for the new clip. + num_inference_steps = request_state.inputs.num_inference_steps + sample_shift = server_args.pipeline_config.flow_shift + self.scheduler.set_timesteps( + sigmas=get_sampling_sigmas(num_inference_steps, sample_shift), device=device + ) + + guidance_scale = request_state.inputs.guidance_scale + + # negative_prompt_embeddings is always computed (an unset prompt encodes ""), so CFG + # depends on the scale alone, unlike Req.validate which also needs a negative prompt. + perform_cfg = guidance_scale > 1 + + # CFG parallel: cond on cfg_rank 0, uncond on cfg_rank 1, then all-gather. Only + # with --enable-cfg-parallel and initialized model-parallel groups. + cfg_parallel = ( + perform_cfg + and bool(server_args.enable_cfg_parallel) + and model_parallel_is_initialized() + ) + if cfg_parallel and get_classifier_free_guidance_world_size() != 2: + logger.warning_once( + "CFG parallel enabled but the CFG group world size is " + f"{get_classifier_free_guidance_world_size()} (Wan-Animate-2 expects 2 " + "for its cond/uncond split); using the sequential two-pass CFG path." + ) + cfg_parallel = False + branches = [ + CFGBranch( + "conditional", + True, + { + "encoder_hidden_states": prompt_embeddings, + "is_unconditional": False, + }, + ) + ] + if perform_cfg: + branches.append( + CFGBranch( + "unconditional", + False, + { + "encoder_hidden_states": negative_prompt_embeddings, + "is_unconditional": True, + }, + ) + ) + cfg_policy = CFGPolicy(branches=branches) + + ctx = WanAnimate2DenoisingContext( + scheduler=self.scheduler, + extra_step_kwargs={}, + target_dtype=torch.bfloat16, + autocast_enabled=True, + timesteps=self.scheduler.timesteps, + num_inference_steps=num_inference_steps, + num_warmup_steps=0, + image_kwargs={}, + pos_cond_kwargs={}, + neg_cond_kwargs={}, + latents=clip_condition.init_noise, + boundary_timestep=None, + z=None, + reserved_frames_mask=None, + seq_len=None, + guidance=None, + is_warmup=batch.is_warmup, + cfg_policy=cfg_policy, + # DPM-Solver advances once per timestep, irrespective of solver order + extra={"progress_step_interval": 1}, + clip_condition=clip_condition, + reference_image_embeddings=reference_image_embeddings, + cfg_parallel=cfg_parallel, + guidance_scale=guidance_scale, + ) + # clip trajectories are not part of the assembled-video output contract + self._run_denoising_loop(ctx, batch, server_args, collect_trajectory=False) + return ctx.latents[:, 1:] + + def _before_denoising_loop( + self, ctx: WanAnimate2DenoisingContext, batch: Req, server_args: ServerArgs + ) -> None: + # activate the DiT before the reference pass, not just the first denoise step + self.begin_declared_component_use( + component_name="transformer", module=self.transformer + ) + with set_forward_context( + current_timestep=0, attn_metadata=None, forward_batch=batch + ): + ctx.reference_kv = self.transformer.build_reference_kv( + clip_cond=ctx.clip_condition + ) + + def _prepare_step_state( + self, + ctx: WanAnimate2DenoisingContext, + batch: Req, + server_args: ServerArgs, + step_index: int, + t_host: torch.Tensor, + timesteps_cpu: torch.Tensor, + ) -> DenoisingStepState: + return DenoisingStepState( + step_index=step_index, + t_host=t_host, + t_device=ctx.timesteps[step_index], + t_int=int(t_host.item()), + current_model=self.transformer, + current_guidance_scale=ctx.guidance_scale, + attn_metadata=None, + ) + + def _run_denoising_step( + self, + ctx: WanAnimate2DenoisingContext, + step: DenoisingStepState, + batch: Req, + server_args: ServerArgs, + ) -> None: + def predict(branch: CFGBranch) -> torch.Tensor: + with set_forward_context( + current_timestep=step.step_index, + attn_metadata=None, + forward_batch=batch, + ): + if branch.name not in ctx.context_by_branch: + ctx.context_by_branch[branch.name] = ( + self.transformer.prepare_context( + branch.kwargs["encoder_hidden_states"], + ctx.reference_image_embeddings, + ) + ) + prediction = self.transformer( + hidden_states=ctx.latents, + timestep=step.t_device.unsqueeze(0), + encoder_hidden_states_image=ctx.reference_image_embeddings, + clip_cond=ctx.clip_condition, + reference_kv=ctx.reference_kv, + projected_context=ctx.context_by_branch[branch.name], + **branch.kwargs, + ) + return prediction.contiguous() if ctx.cfg_parallel else prediction + + predictions = ( + run_cfg_parallel(ctx.cfg_policy, predict) + if ctx.cfg_parallel + else [predict(branch) for branch in ctx.cfg_policy.branches] + ) + noise_pred = predictions[0] + if len(predictions) == 2: + noise_pred = predictions[1] + step.current_guidance_scale * ( + noise_pred - predictions[1] + ) + ctx.latents = ctx.scheduler.step( + noise_pred, + step.t_device, + ctx.latents.unsqueeze(0), + return_dict=False, + )[0].squeeze(0) + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> Req: + request_state = request_state_from_batch(batch) + + output_frames: list[np.ndarray] = [] # [H, W, C] uint8 frames + + # [C, num_frames_conditioning, H, W] bf16 in [-1, 1] + prev_clip_conditioning_frames: torch.Tensor | None = None + + # The DiT runs in bf16 autocast like upstream Wan (timestep and norm math in fp32). + with torch.autocast( + device_type=current_platform.device_type, + dtype=torch.bfloat16, + enabled=True, + ): + for clip_denoising_metadata in request_state.schedule: + is_clip_0 = clip_denoising_metadata.clip_index == 0 + + if not is_clip_0 and prev_clip_conditioning_frames is None: + raise ValueError( + "For all clips except clip-0 `prev_clip_conditioning_frames` must be not None." + ) + clip_condition = build_clip_conditioning( + request_state, + clip_denoising_metadata, + prev_clip_conditioning_frames, + vae_encoder=self._vae_encoder, + image_embedder=self._image_embedder, + ) + + latents = self._denoise_single_clip( + batch, + server_args, + request_state, + clip_condition=clip_condition, + ) + + denoised_latents = latents.to(dtype=torch.float32) + with self.use_declared_component( + component_name="vae", module=self.vae.vae + ): + # [1, C, T, H, W] fp32 in [-1, 1] + decoded_frames = self.clip_decoder.decode_raw( + denoised_latents.unsqueeze(0), + server_args, + vae_dtype=torch.float32, + ) + + # [T, H, W, C] uint8 + output_frames_for_clip = ( + rearrange(((decoded_frames + 1) * 127.5), "1 c t h w -> t h w c") + .detach() + .to(torch.uint8) + .cpu() + .numpy() + ) + + # Drop the overlap frames the previous clip already emitted; 0 for clip 0. + output_frames_for_clip = output_frames_for_clip[ + clip_denoising_metadata.num_frames_conditioning_for_clip : + ] + output_frames.extend(output_frames_for_clip) + + # The next clip takes the last ``num_frames_conditioning`` frames. + prev_clip_conditioning_frames = ( + decoded_frames[ + 0, :, -request_state.inputs.num_frames_conditioning : + ] + .detach() + .to(torch.bfloat16) + ) + # only the independent overlap frames survive into the next clip + del decoded_frames, denoised_latents, latents, clip_condition + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + # Already-decoded [T, H, W, C] uint8; WanAnimate2OutputStage must not re-decode. + request_state.decoded_frames = self._post_process_frames( + output_frames, + request_state.num_reference_video_frames, + request_state.letterbox_info, + ) + return batch + + @staticmethod + def _post_process_frames( + frames: list[np.ndarray], + num_output_frames: int, + letterbox_info: LetterboxInfo, + ) -> np.ndarray: + """Stack ``list[np.ndarray]`` to ``[T, H, W, C]`` uint8, trim to ``num_output_frames`` and crop the + ``resize_by_area`` letterbox.""" + if len(frames) == 0: + raise ValueError( + "WanAnimate2DenoisingStage produced 0 decoded frames to post-process." + ) + frames_np = np.stack(frames, axis=0) # [T, H, W, C] + frames_np = frames_np[:num_output_frames] + + return letterbox_info.crop(frames_np) + + +class WanAnimate2OutputStage(PipelineStage): + """Terminal stage: wrap the already-decoded, assembled frames in an OutputBatch. + + Replaces the standard DecodingStage (which would VAE-decode a single-clip latent). + """ + + @property + def role_affinity(self) -> RoleType: + # The terminal slot of the standard stage layout; disaggregated deployment itself + # is rejected by Wan_Animate_2_14B_Config.supports_disaggregation. + return RoleType.DECODER + + def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: + result = VerificationResult() + request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY) + result.add_check( + "decoded_frames", + ( + request_state.decoded_frames + if isinstance(request_state, WanAnimate2RequestState) + else None + ), + _is_decoded_frames, + ) + return result + + def verify_output( + self, batch: OutputBatch, server_args: ServerArgs + ) -> VerificationResult: + result = VerificationResult() + result.add_check("output", batch.output, V.list_not_empty) + return result + + @torch.no_grad() + def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: + request_state = request_state_from_batch(batch) + frames = request_state.decoded_frames + if frames is None: + raise ValueError( + "WanAnimate2OutputStage: WanAnimate2DenoisingStage did not assemble decoded frames." + ) + # frames: [T, H, W, C] uint8, consumed directly by the save path. Audio, when + # extracted, is [1, C, L] fp32 in [-1, 1] with its sample rate; None gives a silent mp4. + return OutputBatch( + output=[frames], + audio=request_state.audio, + audio_sample_rate=request_state.audio_sample_rate, + metrics=batch.metrics, + ) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py new file mode 100644 index 0000000000..e1457575e1 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Adapters exposing the official Wan encoder API over the diffusers/transformers models the +# sglang loaders produce: image_encoder.visual takes list[torch.Tensor] of [C, T, H, W] and returns +# [B, 257, 1280]; vae.encode/decode fold in the Wan latent scaling. +from __future__ import annotations + +from typing import Any + +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import CLIPImageProcessor + +from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context + + +class WanAnimate2ImageEncoderAdapter: + """Official Wan CLIP API over a ``CLIPVisionModel``: ``visual`` takes ``list[torch.Tensor]`` of ``[C, T, H, W]`` + in [-1, 1] and returns ``hidden_states[-2]`` as ``[B, 257, 1280]`` in the model dtype, the pre-final-LayerNorm + slice the Wan-I2V ``postprocess_image`` consumes.""" + + def __init__( + self, + model: nn.Module, + image_size: int, + image_mean: tuple[float, float, float], + image_std: tuple[float, float, float], + hidden_state_index: int = -2, + device: torch.device | str | None = None, + ) -> None: + self.model = model + self.image_size = image_size + self.image_mean = image_mean + self.image_std = image_std + self.hidden_state_index = hidden_state_index + self.device = device + + @classmethod + def from_loaded( + cls, + model: nn.Module, + image_processor: CLIPImageProcessor, + device: torch.device | str | None = None, + ) -> WanAnimate2ImageEncoderAdapter: + """Build the adapter with image_size / mean / std from the checkpoint's + ``CLIPImageProcessor`` (``crop_size`` is the post-resize crop the model sees).""" + crop_h = int(image_processor.crop_size["height"]) + crop_w = int(image_processor.crop_size["width"]) + if crop_h != crop_w: + raise ValueError( + "WanAnimate2ImageEncoderAdapter expects a square CLIP crop_size; got " + f"height={crop_h}, width={crop_w}." + ) + return cls( + model, + image_size=crop_h, + image_mean=tuple(image_processor.image_mean), + image_std=tuple(image_processor.image_std), + device=device, + ) + + def _resolve_device(self) -> torch.device: + if self.device is not None: + return torch.device(self.device) + return next(self.model.parameters()).device + + @torch.no_grad() + def visual(self, videos: list[torch.Tensor]) -> torch.Tensor: + # The caller holds the component through the residency manager, so the model is + # already where it should be; inputs follow it. Runs in the model's own dtype with + # autocast off: the fused patch-embed conv does not autocast an fp32 weight against + # a bf16 input. + device = self._resolve_device() + model_dtype = next(self.model.parameters()).dtype + size = (self.image_size, self.image_size) + + # Each video [C, T, H, W] becomes T images [T, C, image_size, image_size]; the images + # of all videos are stacked into one batch [N, C, image_size, image_size], N = total + # frame count. + pixels = torch.cat( + [ + F.interpolate( + video.transpose(0, 1).to(device=device, dtype=model_dtype), + size=size, + mode="bicubic", + align_corners=False, + ) + for video in videos + ] + ) + + # [-1, 1] -> [0, 1] then Normalize(mean, std), in the model's own dtype. + mean = torch.tensor(self.image_mean, device=device, dtype=pixels.dtype).view( + 1, 3, 1, 1 + ) + std = torch.tensor(self.image_std, device=device, dtype=pixels.dtype).view( + 1, 3, 1, 1 + ) + pixels = pixels.mul(0.5).add(0.5).sub(mean).div(std) + + # Encode; return the hidden_states[-2] slice. + with ( + torch.autocast(device_type=device.type, enabled=False), + set_forward_context(current_timestep=0, attn_metadata=None), + ): + outputs = self.model(pixel_values=pixels, output_hidden_states=True) + return outputs.hidden_states[self.hidden_state_index] + + +class WanAnimate2VaeAdapter(nn.Module): + """Stage-local latent-scaling adapter over the pipeline's native ``AutoencoderKLWan``. + ``encode`` takes ``list[torch.Tensor]`` (or ``[B, C, T, H, W]``) and returns ``list[torch.Tensor]`` of + ``[16, T', H', W']`` fp32, ``(mode() - latents_mean) / latents_std``; ``decode`` inverts that and returns + ``[B, C, T, H, W]`` fp32 in [-1, 1], which is why ``Wan_Animate_2_14B_Config.get_decode_scale_and_shift`` + returns ``(1.0, None)``.""" + + @property + def use_parallel_decode(self) -> bool: + return self.vae.use_parallel_decode + + def enable_tiling(self) -> None: + self.vae.enable_tiling() + + def __init__(self, vae: nn.Module) -> None: + super().__init__() + if not isinstance(vae, nn.Module): + raise TypeError( + "WanAnimate2VaeAdapter expects an nn.Module AutoencoderKLWan; got " + f"{type(vae).__name__!r}." + ) + self.vae = vae + + # Per-channel latent scaling (length z_dim), which the sglang AutoencoderKLWan + # copies from its config onto the module. Plain (unregistered) tensors, reshaped + # to broadcast over [B, C, T, H, W]. + self._latents_mean = torch.as_tensor( + vae.latents_mean, dtype=torch.float32 + ).view(1, -1, 1, 1, 1) + self._latents_std = torch.as_tensor(vae.latents_std, dtype=torch.float32).view( + 1, -1, 1, 1, 1 + ) + + @staticmethod + def _to_list_of_tensors( + items: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...], + ) -> list[torch.Tensor]: + """Per-item tensors from either a list or a batched ``[B, ...]`` tensor; the stages + pass lists, the shared decode path passes a batch.""" + if isinstance(items, (list, tuple)): + return list(items) + return [items[i] for i in range(items.shape[0])] + + def _mean_std_like(self, input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + return ( + self._latents_mean.to(device=input.device, dtype=input.dtype), + self._latents_std.to(device=input.device, dtype=input.dtype), + ) + + @torch.no_grad() + def encode( + self, + videos: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...], + *args: Any, + **kwargs: Any, + ) -> list[torch.Tensor]: + out: list[torch.Tensor] = [] + for sample in self._to_list_of_tensors(videos): + # Run fp32 with autocast off: the fused causal-conv path has no dtype guard, so + # an outer autocast(bf16) would cast the conv input against the fp32 bias. + # The sglang AutoencoderKLWan returns the DiagonalGaussianDistribution itself, + # not a diffusers AutoencoderKLOutput. + with torch.autocast(device_type=sample.device.type, enabled=False): + posterior = self.vae.encode(sample.unsqueeze(0).float()) + + # Deterministic mu (not a sampled draw). + mu = posterior.mode().float() + mean, std = self._mean_std_like(mu) + z = (mu - mean) / std + out.append(z.squeeze(0)) + return out + + @torch.no_grad() + def decode( + self, + latents: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...], + *args: Any, + **kwargs: Any, + ) -> torch.Tensor: + out: list[torch.Tensor] = [] + for sample in self._to_list_of_tensors(latents): + z = sample.unsqueeze(0).float() + mean, std = self._mean_std_like(z) + z = z * std + mean + + # fp32, autocast OFF: see ``encode`` above. decode returns a plain tensor. + with torch.autocast(device_type=z.device.type, enabled=False): + decoded = self.vae.decode(z) + out.append(decoded.float().clamp_(-1.0, 1.0).squeeze(0)) + + # Clips are decoded individually; avoid copying their full pixel tensor. + return out[0].unsqueeze(0) if len(out) == 1 else torch.stack(out, dim=0) + + def forward( + self, + z: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...], + *args: Any, + **kwargs: Any, + ) -> torch.Tensor: + return self.decode(z, *args, **kwargs) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py new file mode 100644 index 0000000000..6785b76141 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py @@ -0,0 +1,324 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from the Wan-Animate-2 reference implementation (Apache-2.0): +# https://github.com/Wan-Video/Wan-Animate-2 +# +# Keep the original Wan preprocessing numerics; the newer Diffusers modular +# processor is not a numerically interchangeable replacement. The +# zigzag_padding / resize_by_area names are kept for parity with the official helpers; +# make_conditioning_mask is the official get_i2v_mask. +from __future__ import annotations + +import math +from copy import deepcopy +from io import BytesIO +from typing import Literal + +import cv2 +import msgspec +import numpy as np +import torch + +from sglang.srt.utils.common import get_video_bytes + +__all__ = [ + "LetterboxInfo", + "MIN_SINGLE_CLIP_LEN", + "resize_by_area", + "letterbox_resize", + "get_frame_indices", + "get_padding_len", + "single_clip_reference_video_len", + "zigzag_padding", + "make_conditioning_mask", + "read_reference_video_frames", + "write_reference_video", +] + + +class LetterboxInfo(msgspec.Struct, frozen=True): + """Where letterbox_resize put the content inside the padded canvas.""" + + # Axis the padding bars sit on: "width" pads left/right, "height" pads top/bottom. + pad_axis: Literal["width", "height"] + # Bar size in pixels before the content along pad_axis. + padding: int + # Content extent in pixels along pad_axis. + content_len: int + + def crop(self, frames: np.ndarray) -> np.ndarray: + """Cut the bars back out of ``[T, H, W, C]`` frames.""" + start, stop = self.padding, self.padding + self.content_len + if self.pad_axis == "width": + return frames[:, :, start:stop, :] + return frames[:, start:stop, :, :] + + +def resize_by_area( + image: np.ndarray, + target_area: int, + keep_aspect_ratio: bool = True, + divisor: int = 16, + padding_color: tuple[int, int, int] = (0, 0, 0), +) -> tuple[np.ndarray, LetterboxInfo]: + """Resize ``image`` (``[H, W, C]`` uint8) so its area is <= ``target_area``, then letterbox-pad to that size.""" + h, w = image.shape[:2] + aspect_ratio = w / h + if keep_aspect_ratio: + height_target = math.sqrt(target_area / aspect_ratio) + width_target = target_area / height_target + else: + width_target = height_target = math.sqrt(target_area) + width_target, height_target = ( + int((width_target // divisor) * divisor), + int((height_target // divisor) * divisor), + ) + + # Resize (INTER_AREA when shrinking, INTER_LINEAR when enlarging) + # Taken as is from the reference implementation. + interpolation = ( + cv2.INTER_AREA if (width_target * height_target < w * h) else cv2.INTER_LINEAR + ) + + return letterbox_resize( + image, + height=height_target, + width=width_target, + padding_color=padding_color, + interpolation=interpolation, + ) + + +def letterbox_resize( + image: np.ndarray, + height: int = 512, + width: int = 512, + padding_color: tuple[int, int, int] = (0, 0, 0), + interpolation: int = cv2.INTER_LINEAR, +) -> tuple[np.ndarray, LetterboxInfo]: + """Aspect-preserving resize of ``image`` into a ``[height, width, C]`` uint8 canvas with letterbox padding.""" + height_source = image.shape[0] + width_source = image.shape[1] + num_channels_source = image.shape[2] + + canvas = np.zeros((height, width, num_channels_source)) + if num_channels_source == 1: + canvas[:, :, 0] = padding_color[0] + else: + canvas[:, :, 0] = padding_color[0] + canvas[:, :, 1] = padding_color[1] + canvas[:, :, 2] = padding_color[2] + + if (height_source / width_source) > (height / width): + width_target = int(height / height_source * width_source) + img = cv2.resize(image, (width_target, height), interpolation=interpolation) + padding = int((width - width_target) / 2) + if len(img.shape) == 2: + img = img[ + :, :, np.newaxis + ] # grayscale image: add a num_channels_source dimension + canvas[:, padding : padding + width_target, :] = img + pad_axis = "width" + content_len = width_target + else: + height_target = int(width / width_source * height_source) + img = cv2.resize(image, (width, height_target), interpolation=interpolation) + padding = int((height - height_target) / 2) + if len(img.shape) == 2: + img = img[ + :, :, np.newaxis + ] # grayscale image: add a num_channels_source dimension + canvas[padding : padding + height_target, :, :] = img + pad_axis = "height" + content_len = height_target + + canvas = np.uint8(canvas) + return canvas, LetterboxInfo( + pad_axis=pad_axis, padding=padding, content_len=content_len + ) + + +def get_frame_indices( + num_frames_source: int, fps_source: float, num_frames_target: int, fps_target: float +) -> list[int]: + """Source frame index (``list[int]``, nearest frame clipped to the source range) for each of the + ``num_frames_target`` output frames when resampling from ``fps_source`` to ``fps_target``.""" + + times = np.arange(0, num_frames_target) / fps_target + frame_indices_source = np.round(times * fps_source).astype(int) + frame_indices_source = np.clip(frame_indices_source, 0, num_frames_source - 1) + + return frame_indices_source.tolist() + + +# Fewest frames the last clip may add beyond its overlap; a multiple of 4 so it also sits +# on the temporal-VAE grid. Taken as is from the reference. +_MIN_LAST_CLIP_NEW_FRAMES = 28 + + +def get_padding_len( + num_frames: int, clip_len: int, num_frames_conditioning: int = 1 +) -> int: + """Reference-video length after zigzag padding so the last clip is long enough and + 4k-aligned; taken as is from the reference implementation.""" + + remaining = (num_frames - num_frames_conditioning) % ( + clip_len - num_frames_conditioning + ) + # Short leftover: raise it to the minimum. Otherwise: round up to the next multiple + # of 4 (VAE temporal stride); the minimum is itself a multiple of 4. + if remaining < _MIN_LAST_CLIP_NEW_FRAMES: + padding_needed = _MIN_LAST_CLIP_NEW_FRAMES - remaining + else: + padding_needed = 4 - remaining % 4 + + return num_frames + padding_needed + + +# get_padding_len extends every reference video to at least this many frames, so a shorter +# clip_len can never hold the whole video in one clip. +MIN_SINGLE_CLIP_LEN = _MIN_LAST_CLIP_NEW_FRAMES + 1 + + +def single_clip_reference_video_len( + clip_len: int, num_frames_conditioning: int = 1 +) -> int: + """Reference-video frame count that get_padding_len extends to exactly ``clip_len``, + so build_schedule yields one full-length clip; used to size the synthetic warmup video.""" + num_frames = clip_len - 1 + if ( + clip_len < MIN_SINGLE_CLIP_LEN + or get_padding_len(num_frames, clip_len, num_frames_conditioning) != clip_len + ): + raise ValueError( + f"clip_len={clip_len} cannot hold a reference video in a single clip; " + f"the schedule needs clip_len >= {MIN_SINGLE_CLIP_LEN}" + ) + return num_frames + + +def zigzag_padding(array: list, target_len: int) -> list: + """Pad ``array`` up to ``target_len`` by bouncing the read index back and forth (deep copies).""" + if target_len < len(array): + raise ValueError( + f"zigzag_padding only extends: target_len {target_len} < len(array) {len(array)}" + ) + index = 0 + flip = False + target_array = [] + while len(target_array) < target_len: + target_array.append(deepcopy(array[index])) + if flip: + index -= 1 + else: + index += 1 + if index == 0 or index == len(array) - 1: + flip = not flip + return target_array + + +def make_conditioning_mask( + latent_t: int, + latent_h: int, + latent_w: int, + num_prefix_conditioning_frames: int = 1, + device: torch.device | str = "cuda", +) -> torch.Tensor: + """Build the ``[4, latent_t, latent_h, latent_w]`` fp32 i2v conditioning mask; ``num_prefix_conditioning_frames`` is the + conditioned prefix length in pixel frames.""" + + # The VAE maps the first frame to 1 frame in latent space. + # For each subsequent 4 pixel frames it maps them to 1 frame in the latent space. + # Hence, the ``(latent_t - 1) * 4 + 1``. + # VAE: 4k + 1 pixel frames -> k + 1 latent frames + mask = torch.zeros(1, (latent_t - 1) * 4 + 1, latent_h, latent_w, device=device) + + mask[:, :num_prefix_conditioning_frames] = 1 + mask = torch.concat( + [torch.repeat_interleave(mask[:, 0:1], repeats=4, dim=1), mask[:, 1:]], dim=1 + ) # repeat the first pixel frame's flag 4x: (k - 1) * 4 + 1 -> 4 * k pixel slots + + mask = mask.view(1, mask.shape[1] // 4, 4, latent_h, latent_w) + # (1, 4*(k + 1), latent_h, latent_w) -> (1, (k + 1), 4, latent_h, latent_w) + + mask = mask.transpose(1, 2)[0] + # (1, (k + 1), 4, latent_h, latent_w) -> (4, (k + 1), latent_h, latent_w) + return mask + + +def read_reference_video_frames(path: str, fps_target: int) -> np.ndarray: + """Decode the reference video resampled to ``fps_target``; returns ``[N, H, W, 3]`` uint8 RGB. + + Three readers: decord on CPU as the reference implementation does (its GPU decoder + exists only in CUDA builds); decord2 is the arm64 build of the same API; PyAV is the + fallback when neither is installed. + """ + source = ( + BytesIO(get_video_bytes(path)) + if path.startswith(("http://", "https://", "data:")) + else path + ) + try: + from decord import VideoReader + except ImportError: + try: + from decord2 import VideoReader + except ImportError: + VideoReader = None + + if VideoReader is not None: + video_reader = VideoReader(source) + num_frames_source = len(video_reader) + fps_source = video_reader.get_avg_fps() + num_frames_target = int(num_frames_source / fps_source * fps_target) + indices = get_frame_indices( + num_frames_source, fps_source, num_frames_target, fps_target + ) + return video_reader.get_batch(indices).asnumpy() + + import av + + # Decode everything once so num_frames_source matches decord's len() exactly. + with av.open(source) as container: + stream = container.streams.video[0] + fps_source = float(stream.average_rate) + frames_source = [ + frame.to_ndarray(format="rgb24") for frame in container.decode(video=0) + ] + num_frames_source = len(frames_source) + num_frames_target = int(num_frames_source / fps_source * fps_target) + indices = get_frame_indices( + num_frames_source, fps_source, num_frames_target, fps_target + ) + return np.stack([frames_source[i] for i in indices]) + + +def write_reference_video(path: str, frames: np.ndarray, fps: int) -> None: + """Write ``[N, H, W, 3]`` uint8 RGB frames to ``path`` as an mp4 at ``fps`` through the + runtime's video writer (the one request outputs go through), so the synthetic warmup + video is encoded exactly like a request output and decodes with the readers above.""" + from sglang.multimodal_gen.configs.sample.sampling_params import DataType + from sglang.multimodal_gen.runtime.entrypoints.utils import post_process_sample + + try: + post_process_sample(frames, DataType.VIDEO, fps, save_file_path=path) + except Exception as e: + raise RuntimeError( + f"the shared video writer (post_process_sample) could not write the " + f"synthetic reference video {path}: {e}" + ) from e + + +def validate_and_get_single_string(value: str | list[str] | None, name: str) -> str: + """This pipeline produces one video per request, so a batched prompt field, + video path or image path must hold exactly one string.""" + if isinstance(value, list): + if len(value) != 1: + raise ValueError( + f"Wan-Animate-2 takes exactly one '{name}' per request, got {len(value)}" + ) + value = value[0] + if not isinstance(value, str): + raise ValueError( + f"Wan-Animate-2 requires '{name}' to be a string, got {value!r}" + ) + return value diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json index 270ef2b809..53c307eb48 100644 --- a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json +++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json @@ -270,6 +270,18 @@ "ssim_threshold": 0.48, "psnr_threshold": 13.0, "mean_abs_diff_threshold": 45.0 + }, + "wan2_2_animate_2_14b_ref2v": { + "clip_threshold": 0.90, + "ssim_threshold": 0.92, + "psnr_threshold": 24.0, + "mean_abs_diff_threshold": 10.0 + }, + "wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": { + "clip_threshold": 0.90, + "ssim_threshold": 0.92, + "psnr_threshold": 24.0, + "mean_abs_diff_threshold": 10.0 } }, "default_clip_threshold_image": 0.92, diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py index b0b82a5534..d4385f8861 100644 --- a/python/sglang/multimodal_gen/test/server/gpu_cases.py +++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py @@ -39,6 +39,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import ( SANA_WM_TI2V_CI_sampling_params, T2I_sampling_params, T2V_sampling_params, + WAN_2_2_ANIMATE_2_14B_sampling_params, _make_modelopt_ci_case, _with_default_num_gpus, ) @@ -63,6 +64,7 @@ from sglang.multimodal_gen.test.test_utils import ( DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST, DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST, DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST, + DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST, DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST, DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST, DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST, @@ -459,6 +461,23 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [ run_models_api_check=False, run_t2v_input_reference_check=False, ), + # Reference image + driving video -> video. The DiT peaks at 76 GB resident, + # so the 80 GB lane streams it layerwise; this also covers the uncond block-9 skip. + DiffusionTestCase( + "wan2_2_animate_2_14b_ref2v", + DiffusionServerArgs( + model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST, + modality="video", + dit_layerwise_offload=True, + text_encoder_cpu_offload=True, + extras=["--layerwise-offload-components", "transformer"], + ), + WAN_2_2_ANIMATE_2_14B_sampling_params, + # The DiT forward takes a per-clip conditioning container and runs forward_ref, + # so the raw-component harness contract does not apply. + run_perf_check=True, + run_component_accuracy_check=False, + ), # flaky # === Helios T2V === # DiffusionTestCase( @@ -975,6 +994,32 @@ TWO_GPU_CASES = [ model_path=DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST, ), ), + # GT = the 1-GPU frames renamed. With replicated encoders, serial VAE decode and the + # VAE channels_last_3d layout the loader otherwise applies only for num_gpus == 1, + # CFG-parallel output is bit-identical to the 1-GPU case, so both share one ground truth. + DiffusionTestCase( + "wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu", + DiffusionServerArgs( + model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST, + modality="video", + cfg_parallel=True, + # On 80 GB cards the resident DiT leaves no room for the full-shape warmup probe; stream it layerwise (bit-identical output). + dit_layerwise_offload=True, + text_encoder_cpu_offload=True, + extras=[ + "--layerwise-offload-components", + "transformer", + "--encoder-parallel", + "replicate", + "--vae-config.use-parallel-decode", + "false", + ], + env_vars={"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "1"}, + ), + WAN_2_2_ANIMATE_2_14B_sampling_params, + run_perf_check=True, + run_component_accuracy_check=False, + ), DiffusionTestCase( "wan2_2_t2v_a14b_2gpu", DiffusionServerArgs( diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 0eb17fca3a..8b8040d023 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -3524,6 +3524,60 @@ "expected_avg_denoise_ms": 0.0, "expected_median_denoise_ms": 0.0, "estimated_full_test_time_s": 73.7 + }, + "wan2_2_animate_2_14b_ref2v": { + "stages_ms": { + "WanAnimate2BeforeDenoisingStage": 3783.78, + "WanAnimate2DenoisingStage": 94604.96, + "WanAnimate2OutputStage": 0.06 + }, + "denoise_step_ms": { + "0": 25832.83, + "1": 7210.15, + "2": 11187.28, + "3": 8460.7, + "4": 8407.93, + "5": 8409.07, + "6": 8408.07, + "7": 8415.78 + }, + "expected_e2e_ms": 98402.77, + "expected_load_ms": 62901.15, + "expected_avg_denoise_ms": 10791.48, + "expected_median_denoise_ms": 8412.42, + "load_peak_vram_mb": 1776.0, + "runtime_peak_vram_mb": 36322.0, + "load_peak_allocated_mb": 1737.0, + "runtime_peak_allocated_mb": 25748.0, + "estimated_full_test_time_s": 489.9, + "warmup_peak_vram_mb": 14598.0 + }, + "wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": { + "stages_ms": { + "WanAnimate2BeforeDenoisingStage": 5778.62, + "WanAnimate2DenoisingStage": 58196.97, + "WanAnimate2OutputStage": 0.06 + }, + "denoise_step_ms": { + "0": 21723.06, + "1": 3125.83, + "2": 4335.41, + "3": 4337.63, + "4": 4347.53, + "5": 4339.76, + "6": 4350.68, + "7": 4280.35 + }, + "expected_e2e_ms": 63988.59, + "expected_load_ms": 64309.0, + "expected_avg_denoise_ms": 6355.03, + "expected_median_denoise_ms": 4338.7, + "load_peak_vram_mb": 1776.0, + "runtime_peak_vram_mb": 36322.0, + "load_peak_allocated_mb": 1737.0, + "runtime_peak_allocated_mb": 25745.0, + "estimated_full_test_time_s": 404.0, + "warmup_peak_vram_mb": 14598.0 } } } diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py index 19b811a40c..7bb78d0258 100644 --- a/python/sglang/multimodal_gen/test/server/testcase_configs.py +++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py @@ -36,6 +36,10 @@ from sglang.multimodal_gen.registry import ( ) from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord +from sglang.multimodal_gen.test.test_utils import ( + SGL_TEST_FILES_CI_DATA_REPO, + SGL_TEST_FILES_CI_DATA_REVISION, +) @dataclass @@ -885,6 +889,35 @@ TURBOWAN_I2V_sampling_params = DiffusionSamplingParams( fps=4, ) +# Wan-Animate-2 inputs; they ship in the same ci-data-diffusion commit as the GT, +# so the pinned revision must point at a commit that carries them. +_WAN_2_2_ANIMATE_2_CI_INPUT_BASE = ( + "https://raw.githubusercontent.com/" + f"{SGL_TEST_FILES_CI_DATA_REPO}/{SGL_TEST_FILES_CI_DATA_REVISION}/" + "diffusion-ci/inputs/wan_animate_2" +) +WAN_2_2_ANIMATE_2_14B_sampling_params = DiffusionSamplingParams( + prompt=( + "Character appearance description: a smiling young woman with shoulder-length " + "dark hair, a mustard-yellow knit sweater, blue jeans and white sneakers. " + "Background description: a plain light-gray studio backdrop." + ), + image_path=f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/reference_short.jpg", + # The model reads `size` as an area budget and emits the 16-aligned canvas fitted to + # the reference aspect; 576x1024 (9:16, like the image) maps onto itself. + output_size="576x1024", + fps=16, + output_format="mp4", + # 24 driving frames at 16 fps pad to one 29-frame clip (get_padding_len), + # so the request denoises a single clip; output length follows the driving video. + extras={ + "video_url": f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/driving_video_short.mp4", + "clip_len": 37, + "num_inference_steps": 8, + "seed": 42, + }, +) + HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams( prompt="", image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png", diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py index df0e4ba14b..6b7cde1725 100644 --- a/python/sglang/multimodal_gen/test/test_utils.py +++ b/python/sglang/multimodal_gen/test/test_utils.py @@ -37,7 +37,7 @@ logger = init_logger(__name__) # NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows # publish. SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion" -SGL_TEST_FILES_CI_DATA_REVISION = "17312c8df37e65bffe75818884bc2a5c93cf3094" +SGL_TEST_FILES_CI_DATA_REVISION = "31d8ad69f2434727aad518f0678fbd57823856fe" # The NPU pin is kept as a separate branch so ascend GT can be bumped independently # when it's regenerated on its own cadence. @@ -200,6 +200,9 @@ DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST = ( DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-TI2V-5B-Diffusers" DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-I2V-A14B-Diffusers" +DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST = ( + "Wan-AI/Wan2.2-Animate-2-14B-Diffusers" +) # MOVA video generation models DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST = "OpenMOSS-Team/MOVA-360p" diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py new file mode 100644 index 0000000000..8cb3af6d0b --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py @@ -0,0 +1,1763 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the Wan-Animate-2 configs, registry wiring, request handling and the +preprocessing helpers that feed the conditioning tensors. + +No models, GPU, or network: the registry detector tests patch +``maybe_download_model_index``; stages run with stand-in encoders. +""" + +import logging +import math +import os +from dataclasses import fields, replace +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np +import pytest +import torch +from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution +from transformers import BatchEncoding + +from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import ( + WanAnimate2ArchConfig, + WanAnimate2Config, +) +from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan_Animate_2_14B_Config +from sglang.multimodal_gen.configs.sample.sampling_params import ( + DataType, + SamplingParams, +) +from sglang.multimodal_gen.configs.sample.wan import Wan_Animate_2_14B_SamplingParam +from sglang.multimodal_gen.registry import _get_config_info +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import ( + VideoGenerationsRequest, +) +from sglang.multimodal_gen.runtime.entrypoints.openai.utils import ( + get_sampling_request_extra_fields, +) +from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import ( + AttentionBackendEnum, +) +from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod +from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import ( + _supported_init_kwargs, +) +from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import ( + TransformerLoader, +) +from sglang.multimodal_gen.runtime.loader.utils import ( + component_name_to_loader_cls, + get_param_names_mapping, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import ( + WanAnimate2Transformer3DModel, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import ( + WanAnimate2TransformerBlock, +) +from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import ( + DPMSolverMultistepScheduler, +) +from sglang.multimodal_gen.runtime.pipelines.wan_animate_2_pipeline import ( + WanAnimate2Pipeline, + WanAnimate2TransformerLoader, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import ( + WanAnimate2BeforeDenoisingStage, + WanAnimate2RequestState, + _WanAnimate2Inputs, + build_clip_conditioning, + build_schedule, + generation_token_grid, + request_state_from_batch, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import ( + WanAnimate2DenoisingStage, + get_sampling_sigmas, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import ( + WanAnimate2VaeAdapter, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import ( + LetterboxInfo, + get_frame_indices, + get_padding_len, + letterbox_resize, + make_conditioning_mask, + read_reference_video_frames, + resize_by_area, + write_reference_video, + zigzag_padding, +) +from sglang.multimodal_gen.runtime.server_args import ServerArgs, set_global_server_args +from sglang.multimodal_gen.runtime.server_warmup import prepare_warmup_image_path +from sglang.multimodal_gen.runtime.warmup_request_builder import ( + SERVER_WARMUP_VIDEO_STEPS, + build_warmup_reqs, + lighten_warmup_req, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( + ensure_distributed_env_defaults, +) + +# Wan_Animate_2_14B_Config + + +def test_pipeline_local_loader_preserves_shared_transformer_registry(): + for name in TransformerLoader.component_names: + assert component_name_to_loader_cls[name] is TransformerLoader + assert not TransformerLoader.strict_checkpoint_keys + assert WanAnimate2TransformerLoader.strict_checkpoint_keys + assert WanAnimate2Pipeline.component_loaders == { + "transformer": WanAnimate2TransformerLoader + } + + +def test_text_encoder_counts_tokens_before_device_transfer(): + config = Wan_Animate_2_14B_Config() + tokens = BatchEncoding( + { + "input_ids": torch.tensor([[1, 2, 0, 0]]), + "attention_mask": torch.tensor([[1, 1, 0, 0]]), + } + ) + inputs = config.tokenize_prompt(["prompt"], lambda *args, **kwargs: tokens, {}) + inputs.to("meta") + assert ( + inputs["input_ids"].device.type + == inputs["attention_mask"].device.type + == "meta" + ) + assert inputs["num_tokens"] == 2 + output = SimpleNamespace(last_hidden_state=torch.empty(1, 4, 8, device="meta")) + assert config.postprocess_text_funcs[0](output, inputs).shape == (2, 8) + assert not WanAnimate2BeforeDenoisingStage.has_deduplicated_output_fields() + + +def test_pipeline_keeps_native_modules_visible_to_memory_managers(): + path = "sglang.multimodal_gen.runtime.pipelines.wan_animate_2_pipeline" + vae = torch.nn.Identity() + vae.latents_mean, vae.latents_std = [0.0], [1.0] + modules = { + "text_encoder": torch.nn.Identity(), + "image_encoder": torch.nn.Identity(), + "vae": vae, + "tokenizer": object(), + "image_processor": SimpleNamespace( + crop_size={"height": 224, "width": 224}, + image_mean=(0.5, 0.5, 0.5), + image_std=(0.5, 0.5, 0.5), + ), + "transformer": object(), + "scheduler": object(), + } + stub = SimpleNamespace( + get_module=modules.__getitem__, + add_stage=lambda **kwargs: None, + add_stage_factory=lambda *args: None, + ) + with ( + patch(f"{path}.get_local_torch_device", return_value=torch.device("cpu")), + patch(f"{path}.WanAnimate2BeforeDenoisingStage") as before, + patch(f"{path}.WanAnimate2DenoisingStage") as denoise, + ): + WanAnimate2Pipeline.create_pipeline_stages( + stub, SimpleNamespace(pipeline_config=Wan_Animate_2_14B_Config()) + ) + args = before.call_args.kwargs + assert "load_modules" not in vars(WanAnimate2Pipeline) + assert args["text_encoder"] is modules["text_encoder"] + assert args["tokenizer"] is modules["tokenizer"] + assert args["image_encoder"].model is modules["image_encoder"] + assert args["vae"].vae is vae + assert denoise.call_args.kwargs["vae"] is args["vae"] + + +def test_single_expert_has_no_boundary_switching(): + # Single-expert model: no low-noise expert switch via boundary_ratio. + config = Wan_Animate_2_14B_Config() + assert config.boundary_ratio is None + assert config.dit_config.arch_config.boundary_ratio is None + + +@pytest.mark.parametrize( + "tp_size, ulysses_degree, enable_cfg_parallel", + [ + (2, 2, False), # TP2 x Ulysses2 (4 GPUs) + (2, 2, True), # TP2 x Ulysses2 auto-derived from --num-gpus 8 with CFG + (2, 1, False), # TP2 alone + (1, 2, False), # Ulysses2 alone + (4, 1, False), # TP4 + (1, 2, True), # CFG-parallel x Ulysses2 + (2, 1, True), # CFG-parallel x TP2 + ], +) +def test_validate_server_args_accepts_tp_combined_with_ulysses( + tp_size, ulysses_degree, enable_cfg_parallel +): + """TP, Ulysses and CFG-parallel compose freely as long as the degrees divide the + 40 attention heads; the hybrid TP x Ulysses layout starts like the single-axis ones.""" + server_args = _parallel_server_args( + tp_size=tp_size, + ulysses_degree=ulysses_degree, + enable_cfg_parallel=enable_cfg_parallel, + ) + Wan_Animate_2_14B_Config().validate_server_args(server_args) + + +def _parallel_server_args( + *, + tp_size: int = 1, + ulysses_degree: int = 1, + ring_degree: int = 1, + enable_cfg_parallel: bool = False, + use_fsdp_inference: bool = False, + attention_backend: str | None = None, + attention_backend_explicit: bool = False, + component_attention_backends: dict[str, str] | None = None, +) -> SimpleNamespace: + """The normalized ServerArgs fields validate_server_args reads (sp_degree = ulysses * ring).""" + requested = dict(component_attention_backends or {}) + explicit = {"attention_backend"} if attention_backend_explicit else set() + return SimpleNamespace( + tp_size=tp_size, + sp_degree=ulysses_degree * ring_degree, + ulysses_degree=ulysses_degree, + ring_degree=ring_degree, + enable_cfg_parallel=enable_cfg_parallel, + use_fsdp_inference=use_fsdp_inference, + attention_backend=attention_backend, + is_arg_explicitly_set=lambda name: name in explicit, + requested_component_attention_backend=lambda name: requested.get(name), + ) + + +def test_validate_server_args_rejects_fsdp_before_any_weights_load(): + with pytest.raises(NotImplementedError, match="use-fsdp-inference"): + Wan_Animate_2_14B_Config().validate_server_args( + _parallel_server_args(use_fsdp_inference=True) + ) + + +def test_validate_server_args_rejects_ring_sequence_parallelism(): + with pytest.raises(NotImplementedError, match="ring sequence parallelism"): + Wan_Animate_2_14B_Config().validate_server_args( + _parallel_server_args(ulysses_degree=1, ring_degree=2) + ) + + +@pytest.mark.parametrize( + "tp_size, ulysses_degree, rejected", + [ + (1, 8, False), # 40 heads / 8 + (1, 3, True), # 40 heads not divisible by 3 + (1, 16, True), # 40 heads not divisible by 16 + (3, 1, True), # 40 heads not divisible by tp 3 + (8, 1, False), + (2, 4, False), # 40 heads / tp 2 = 20 local heads, divisible by 4 + (4, 2, False), # 40 heads / tp 4 = 10 local heads, divisible by 2 + (4, 3, True), # 10 local heads not divisible by 3 + (8, 4, True), # 5 local heads not divisible by 4 + ], +) +def test_validate_server_args_checks_head_divisibility( + tp_size, ulysses_degree, rejected +): + """Ulysses splits the TP-local heads across ranks; a non-dividing degree must be + rejected at startup, before any weights load.""" + server_args = _parallel_server_args(tp_size=tp_size, ulysses_degree=ulysses_degree) + config = Wan_Animate_2_14B_Config() + if rejected: + with pytest.raises(ValueError, match="divisible by"): + config.validate_server_args(server_args) + else: + config.validate_server_args(server_args) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"attention_backend": "fa", "attention_backend_explicit": True}, + {"component_attention_backends": {"transformer": "torch_sdpa"}}, + ], +) +def test_validate_server_args_rejects_a_requested_dit_attention_backend(kwargs): + # The in-context self-attention is flex_attention only, so a requested backend + # would be honoured by the cross-attention and silently dropped by the main path. + with pytest.raises(ValueError, match="flex_attention only"): + Wan_Animate_2_14B_Config().validate_server_args(_parallel_server_args(**kwargs)) + + +def test_validate_server_args_keeps_a_platform_default_attention_backend(): + # ROCm fills attention_backend in when the user gave none; that is not a request. + Wan_Animate_2_14B_Config().validate_server_args( + _parallel_server_args( + attention_backend="aiter", attention_backend_explicit=False + ) + ) + + +def test_validate_server_args_rejects_mxfp8_offline_qk_rotation(monkeypatch): + monkeypatch.setenv("SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION", "true") + with pytest.raises(ValueError, match="MXFP8"): + Wan_Animate_2_14B_Config().validate_server_args(_parallel_server_args()) + + +class _RotatedQuantConfig: + """A quantized checkpoint that ships offline Q/K rotation matrices for one block.""" + + def __init__(self, prefix: str) -> None: + self.quant_description = { + f"{prefix}.attn1.q_rot": "FLOAT", + f"{prefix}.attn1.k_rot": "FLOAT", + } + + def get_quant_method(self, layer, prefix: str) -> UnquantizedLinearMethod: + return UnquantizedLinearMethod() + + +def test_block_refuses_a_checkpoint_with_offline_qk_rotation(monkeypatch): + # The parent block registers q_rot / k_rot and applies them in its own attention + # path; forward_ref / forward_gen never do, so loading would be silently wrong. + monkeypatch.setenv("SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION", "true") + _ensure_single_process_parallel_runtime() + with torch.device("meta"), pytest.raises(ValueError, match="offline Q/K rotation"): + WanAnimate2TransformerBlock( + 64, + 128, + 2, + "rms_norm", + True, + 1e-6, + None, + {AttentionBackendEnum.TORCH_SDPA}, + prefix="blocks.0", + quant_config=_RotatedQuantConfig("blocks.0"), + ) + + +# WanAnimate2ArchConfig / WanAnimate2Config + + +def test_in_out_channels_match_in_context_io(): + # in_channels = 16 noise + 20 conditioning; out_channels = 16 latent. + arch = WanAnimate2ArchConfig() + assert arch.in_channels == 36 + assert arch.out_channels == 16 + + +def test_backbone_dims_reuse_wan22_i2v_14b(): + arch = WanAnimate2ArchConfig() + assert arch.num_layers == 40 + assert arch.image_dim == 1280 + assert arch.added_kv_proj_dim == 5120 + + +def test_post_init_derives_hidden_size_and_latent_channels(): + arch = WanAnimate2ArchConfig() + assert arch.num_attention_heads == 40 + assert arch.attention_head_dim == 128 + assert arch.hidden_size == 5120 + assert arch.num_channels_latents == 16 + + +# The 1303 tensor names of Wan2.2-Animate-2-14B-Diffusers/transformer (the safetensors +# index's weight_map): 23 top-level tensors plus 32 per block for 40 blocks. +_CHECKPOINT_TOP_LEVEL_KEYS = ( + "head.head.bias", + "head.head.weight", + "head.modulation", + "img_emb.proj.0.bias", + "img_emb.proj.0.weight", + "img_emb.proj.1.bias", + "img_emb.proj.1.weight", + "img_emb.proj.3.bias", + "img_emb.proj.3.weight", + "img_emb.proj.4.bias", + "img_emb.proj.4.weight", + "patch_embedding.bias", + "patch_embedding.weight", + "text_embedding.0.bias", + "text_embedding.0.weight", + "text_embedding.2.bias", + "text_embedding.2.weight", + "time_embedding.0.bias", + "time_embedding.0.weight", + "time_embedding.2.bias", + "time_embedding.2.weight", + "time_projection.1.bias", + "time_projection.1.weight", +) +_CHECKPOINT_BLOCK_KEYS = ( + "cross_attn.add_k_proj.bias", + "cross_attn.add_k_proj.weight", + "cross_attn.add_v_proj.bias", + "cross_attn.add_v_proj.weight", + "cross_attn.norm_added_k.weight", + "cross_attn.norm_k.weight", + "cross_attn.norm_q.weight", + "cross_attn.to_k.bias", + "cross_attn.to_k.weight", + "cross_attn.to_out.0.bias", + "cross_attn.to_out.0.weight", + "cross_attn.to_q.bias", + "cross_attn.to_q.weight", + "cross_attn.to_v.bias", + "cross_attn.to_v.weight", + "ffn.0.bias", + "ffn.0.weight", + "ffn.2.bias", + "ffn.2.weight", + "modulation", + "norm3.bias", + "norm3.weight", + "self_attn.norm_k.weight", + "self_attn.norm_q.weight", + "self_attn.to_k.bias", + "self_attn.to_k.weight", + "self_attn.to_out.0.bias", + "self_attn.to_out.0.weight", + "self_attn.to_q.bias", + "self_attn.to_q.weight", + "self_attn.to_v.bias", + "self_attn.to_v.weight", +) + + +def _checkpoint_keys() -> list[str]: + keys = list(_CHECKPOINT_TOP_LEVEL_KEYS) + for block in range(WanAnimate2ArchConfig().num_layers): + keys.extend(f"blocks.{block}.{key}" for key in _CHECKPOINT_BLOCK_KEYS) + return keys + + +def test_every_checkpoint_key_maps_onto_a_distinct_dit_parameter(): + """The DiT loads with the shared Wan rules plus the Wan-Animate-2 extras; a rule that + stops matching leaves a tensor unmapped (skipped with a warning) or lands two tensors on + one parameter, both of which load silently wrong weights.""" + keys = _checkpoint_keys() + assert len(keys) == 1303 + map_fn = get_param_names_mapping(WanAnimate2ArchConfig().param_names_mapping) + mapped = {key: map_fn(key)[0] for key in keys} + unmapped = sorted(key for key, name in mapped.items() if name == key) + assert unmapped == [] + assert len(set(mapped.values())) == len(keys) + + _ensure_single_process_parallel_runtime() + with torch.device("meta"): + model = WanAnimate2Transformer3DModel(WanAnimate2Config(), hf_config={}) + assert set(mapped.values()) == set(model.state_dict()) + + +def _ensure_single_process_parallel_runtime() -> None: + # The DiT's parallel linear layers need the TP group even on a meta device. + if model_parallel_is_initialized(): + return + ensure_distributed_env_defaults() + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +def test_official_checkpoint_config_names_update_the_arch_config(): + """transformer/config.json carries the official names (dim, in_dim, num_heads, ...); + the loader's update_model_arch must land them on the fields the model reads.""" + config = WanAnimate2Config() + config.update_model_arch( + { + "dim": 5120, + "in_dim": 36, + "out_dim": 16, + "num_heads": 40, + "patch_size": [1, 2, 2], + "refer_offset_t": 2, + "log_scale": -1.3, + "text_len": 512, + } + ) + assert config.num_attention_heads == 40 + assert config.attention_head_dim == 128 + assert config.hidden_size == 5120 + assert config.in_channels == 36 + assert config.out_channels == 16 + assert config.patch_size == (1, 2, 2) + assert config.refer_offset_t == 2 + assert config.log_scale == -1.3 + assert config.text_len == 512 + + +def test_text_len_defaults_to_the_checkpoint_value(): + """The pipeline pads prompts to ``arch_config.text_len``: the checkpoint's + transformer/config.json sets it and the default covers a config without the key.""" + assert WanAnimate2ArchConfig().text_len == 512 + assert Wan_Animate_2_14B_Config().dit_config.arch_config.text_len == 512 + + +def test_checkpoint_config_without_image_embedding_is_rejected(): + # The model always builds the image-embedding cross-attention; a checkpoint that + # declares use_img_emb false would load and run the wrong architecture silently. + config = WanAnimate2Config() + with pytest.raises(ValueError, match="use_img_emb"): + config.update_model_arch({"use_img_emb": False}) + + +# Wan_Animate_2_14B_SamplingParam + + +@pytest.mark.parametrize( + "requested, expected", + [ + (37, 37), # 4k+1 -> unchanged + (80, 81), # 4k -> up one frame + (42, 41), # 4k+2 -> down + (43, 41), # 4k+3 -> down + (4, 5), # smallest accepted clip + ], +) +def test_clip_len_rounds_to_4k_plus_1(requested, expected, caplog): + # VAE temporal stride is 4, so a clip must be 4k+1 frames; the rounding is logged + # once, and only when it changed the value. + with caplog.at_level(logging.INFO): + params = Wan_Animate_2_14B_SamplingParam(clip_len=requested) + assert params.clip_len == expected + assert params.clip_len % 4 == 1 + rounding_logs = [r for r in caplog.records if "is not 4k+1" in r.getMessage()] + assert len(rounding_logs) == (0 if requested == expected else 1) + if rounding_logs: + assert f"clip_len={requested}" in rounding_logs[0].getMessage() + assert f"clip_len={expected}" in rounding_logs[0].getMessage() + + +@pytest.mark.parametrize("requested", [-3, 0, 1, 2, 3]) +def test_clip_len_below_one_temporal_stride_is_rejected(requested): + # A clip shorter than one VAE temporal stride is rejected at admission, not in a stage. + with pytest.raises(ValueError, match=rf"clip_len={requested} is not supported"): + Wan_Animate_2_14B_SamplingParam(clip_len=requested) + + +def test_num_frames_is_not_mirrored_from_clip_len(): + """The output length follows the reference video, so clip_len must not be written + into num_frames: a mirrored value made `dataclasses.replace(num_frames=...)` snap back, + and the auto-residency probe (which shrinks a request by replacing num_frames until its + estimate fits) looped forever on cards where the full probe did not fit.""" + params = Wan_Animate_2_14B_SamplingParam(clip_len=37) + assert params.num_frames == SamplingParams.num_frames + assert params.adjust_frames is False + + lighter = replace(Wan_Animate_2_14B_SamplingParam(num_frames=37), num_frames=17) + assert lighter.num_frames == 17 + assert lighter.clip_len == 37 + + +def test_lighten_warmup_req_shrinks_a_wan_animate_2_probe(tmp_path): + # Each shrink step must strictly reduce width*height*num_frames, the quantity the + # auto-residency probe loop terminates on; the ladder bottoms out at the 16-pixel floor. + server_args = _server_args(str(tmp_path)) + req = Req( + sampling_params=Wan_Animate_2_14B_SamplingParam(num_frames=37), + prompt="probe", + width=640, + height=800, + ) + units = [] + for _ in range(64): + params = req.sampling_params + units.append(params.width * params.height * params.num_frames) + req = lighten_warmup_req(server_args, req) + if req is None: + break + assert req is None, "the shrink ladder never bottomed out" + assert units == sorted(units, reverse=True) and len(set(units)) == len(units) + + +def test_multi_gpu_frame_alignment_leaves_num_frames_alone(tmp_path, caplog): + # The stages read clip_len, so the multi-GPU frame alignment must not touch num_frames. + server_args = _server_args(str(tmp_path)) + server_args.num_gpus = 4 + server_args.comfyui_mode = True + params = Wan_Animate_2_14B_SamplingParam( + prompt="p", image_path="ref.png", video_path="drv.mp4", num_frames=37 + ) + with caplog.at_level(logging.INFO): + params._adjust_visual_fields(server_args, server_args.pipeline_config) + assert params.num_frames == 37 + messages = [r.getMessage() for r in caplog.records] + assert not any("based on number of GPUs" in m for m in messages) + assert any("ignores num_frames=37" in m for m in messages) + + +@pytest.mark.parametrize("field_name", ["enable_teacache", "enable_spectrum"]) +def test_sampling_params_reject_the_shared_dit_cache_heuristics(field_name): + # Every block runs at every step in the in-context loop; the flags would be accepted + # and change nothing. + with pytest.raises(ValueError, match="does not support"): + Wan_Animate_2_14B_SamplingParam(**{field_name: True}) + + +# Registry resolution (no network) + + +def test_registered_hf_path_resolves_to_wan_animate_configs(): + info = _get_config_info("Wan-AI/Wan2.2-Animate-2-14B-Diffusers") + assert info is not None + assert info.pipeline_config_cls is Wan_Animate_2_14B_Config + assert info.sampling_param_cls is Wan_Animate_2_14B_SamplingParam + + +@pytest.mark.parametrize( + "path", ["some/local/dir/wan_animate_2", "my-wan2.2-animate-2-ckpt"] +) +def test_detector_resolves_wan_animate_style_paths(path): + with patch( + "sglang.multimodal_gen.registry.maybe_download_model_index", + return_value={}, + ): + info = _get_config_info(path) + assert info is not None + assert info.pipeline_config_cls is Wan_Animate_2_14B_Config + assert info.sampling_param_cls is Wan_Animate_2_14B_SamplingParam + + +def test_model_index_class_name_routes_a_checkpoint_at_any_path(): + # A local checkpoint directory need not carry a model token in its name; its + # model_index.json _class_name is what identifies it. + with patch( + "sglang.multimodal_gen.registry.maybe_download_model_index", + return_value={"_class_name": "WanAnimate2Pipeline"}, + ): + info = _get_config_info("/models/ckpt-7f3a") + assert info is not None + assert info.pipeline_config_cls is Wan_Animate_2_14B_Config + + +def test_v1_animate_checkpoint_does_not_route_to_wan_animate_2(): + # Wan2.2-Animate-14B (v1) is a different model and pipeline; a detector loosened to + # "animate" would pass the positive cases above and mis-route it here. + with patch( + "sglang.multimodal_gen.registry.maybe_download_model_index", + return_value={"_class_name": "WanAnimatePipeline"}, + ): + info = _get_config_info("Wan-AI/Wan2.2-Animate-14B") + assert info is None or info.pipeline_config_cls is not Wan_Animate_2_14B_Config + + +# /v1/videos routing: the model-specific knobs are declared extras, the reference image and +# driving video travel in the base request fields. + + +def test_video_api_routes_the_model_knobs_as_declared_extra_fields(): + extras = get_sampling_request_extra_fields(Wan_Animate_2_14B_SamplingParam, "video") + assert extras == {"clip_len", "prompt_ref", "enable_audio"} + base_fields = {field.name for field in fields(SamplingParams)} + for name in extras: + # Not a base request field (no silent duplicate), and not on the base params + # either, so the declaration is what makes the API forward the value. + assert name not in VideoGenerationsRequest.model_fields, name + assert name not in base_fields, name + for name in ("input_reference", "reference_url", "video_path", "video_url"): + assert name in VideoGenerationsRequest.model_fields, name + + +# Preprocessing helpers (preprocess.py): their numerics feed the conditioning tensors and have +# no diffusers equivalent, so exact outputs are pinned. + +# get_frame_indices(num_frames_source, fps_source, num_frames_target, fps_target): +# idx = clip(round(arange(num_frames_target) / fps_target * fps_source), 0, num_frames_source - 1) + + +@pytest.mark.parametrize( + "args, expected", + [ + ((100, 16, 5, 16), [0, 1, 2, 3, 4]), # fps match: identity + ((100, 30, 5, 15), [0, 2, 4, 6, 8]), # downsample by the fps ratio + ((3, 30, 6, 15), [0, 2, 2, 2, 2, 2]), # clipped to the last available frame + ], +) +def test_get_frame_indices_resamples_by_fps_ratio(args, expected): + assert get_frame_indices(*args) == expected + + +def test_get_frame_indices_uses_banker_rounding(): + # times*video_fps = [0.0, 0.5, 1.0, 1.5]; np.round is half-to-even, so 0.5 -> 0 and + # 1.5 -> 2. Python round()/int() truncation would shift the sampled frames. + assert get_frame_indices(10, 1, 4, 2) == [0, 0, 1, 2] + + +# get_padding_len(num_frames, clip_len, num_frames_conditioning=1): +# remaining = (num_frames - num_frames_conditioning) % (clip_len - num_frames_conditioning) +# pad = 28 - remaining if remaining < 28 else 4 - remaining % 4 + +_PADDING_GOLDENS = { + 1: 29, # remaining 0 -> pad 28 + 28: 29, # remaining 27 -> pad 1 (boundary, still < 28) + 29: 33, # remaining 28 -> else branch, pad 4 + 32: 33, # remaining 31 -> pad 1 + 50: 53, # remaining 49 -> pad 3 + 81: 109, # remaining 0 -> pad 28 + 100: 109, # remaining 19 -> pad 9 +} + + +def test_get_padding_len_matches_reference_goldens(): + for input_len, expected in _PADDING_GOLDENS.items(): + assert get_padding_len(input_len, 81) == expected, input_len + + +@pytest.mark.parametrize("clip_len", [37, 81]) +def test_get_padding_len_output_is_4k_plus_1(clip_len): + # Padded length must land on a 4k+1 boundary (VAE temporal stride 4) for the shipped + # default clip length and the official one. + for input_len in range(1, 200): + assert (get_padding_len(input_len, clip_len) - 1) % 4 == 0, input_len + + +# build_schedule(reference_video_frames, clip_len, num_frames_conditioning) +# -> [_PerClipDenoisingMetadata]; frame_end_index is exclusive. + + +def _frames(num_frames: int, height: int = 1, width: int = 1) -> list[np.ndarray]: + return [ + np.full((height, width, 3), index % 256, np.uint8) + for index in range(num_frames) + ] + + +def _as_tuples(schedule): + return [ + ( + clip.clip_index, + clip.frame_start_index, + clip.frame_end_index, + clip.num_frames, + clip.num_frames_conditioning_for_clip, + ) + for clip in schedule + ] + + +@pytest.mark.parametrize( + "num_frames, clip_len, num_frames_conditioning, expected", + [ + # Exact tiling: 37 + 2 * (37 - 1) = 109; every clip after the first reuses one frame. + (109, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 73, 37, 1), (2, 72, 109, 37, 1)]), + # Last clip shrinks to the frames left. + (50, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 50, 14, 1)]), + # Frame 72 is already covered by clip 1, so no 1-frame clip is appended... + (73, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 73, 37, 1)]), + # ...but one extra new frame does produce a clip. + (74, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 73, 37, 1), (2, 72, 74, 2, 1)]), + # Overlap follows num_frames_conditioning, not a hardcoded 1. + ( + 14, + 5, + 2, + [(0, 0, 5, 5, 0), (1, 3, 8, 5, 2), (2, 6, 11, 5, 2), (3, 9, 14, 5, 2)], + ), + ], +) +def test_build_schedule_clip_boundaries( + num_frames, clip_len, num_frames_conditioning, expected +): + schedule = build_schedule(_frames(num_frames), clip_len, num_frames_conditioning) + assert _as_tuples(schedule) == expected + covered = set() + for clip in schedule: + covered.update(range(clip.frame_start_index, clip.frame_end_index)) + assert covered == set(range(num_frames)) + + +@pytest.mark.parametrize("num_frames", [5, 37, 50, 100, 236]) +def test_build_schedule_after_padding_yields_4k_plus_1_clips(num_frames): + # Same call chain as WanAnimate2BeforeDenoisingStage: pad first, then schedule. + clip_len = 37 + padded_len = get_padding_len(num_frames, clip_len) + schedule = build_schedule( + zigzag_padding(_frames(num_frames), padded_len), clip_len, 1 + ) + assert schedule[-1].frame_end_index == padded_len + assert all(clip.num_frames % 4 == 1 for clip in schedule) + assert all(clip.num_frames == clip_len for clip in schedule[:-1]) + # get_padding_len's reason to exist: the last clip generates at least 28 new frames. + assert schedule[-1].num_frames - 1 >= 28 + + +def test_build_schedule_rejects_video_too_short_for_a_clip(): + with pytest.raises(ValueError, match="0 clips"): + build_schedule(_frames(1), 37, 1) + + +# zigzag_padding(array, target_len): bounce the read index 0->end->0 to pad up to target_len. + + +@pytest.mark.parametrize( + "array, target_len, expected", + [ + ([0, 1, 2], 8, [0, 1, 2, 1, 0, 1, 2, 1]), + (["a", "b"], 5, ["a", "b", "a", "b", "a"]), # bounces at both ends every step + ], +) +def test_zigzag_padding_bounces_forward_then_backward(array, target_len, expected): + assert zigzag_padding(array, target_len) == expected + + +def test_zigzag_padding_rejects_shortening(): + with pytest.raises(ValueError): + zigzag_padding([0, 1, 2], 2) + + +def test_zigzag_padding_returns_independent_deep_copies(): + # A shallow copy would alias reused frames across the zigzag. + source = [[0], [1]] + out = zigzag_padding(source, 4) + out[0][0] = 99 + assert source == [[0], [1]] + + +# make_conditioning_mask(latent_t, latent_h, latent_w, num_prefix_conditioning_frames, device) +# -> [4, latent_t, latent_h, latent_w]; the prefix is counted in pixel frames and the VAE +# maps 4k+1 pixel frames to k+1 latent frames. + + +@pytest.mark.parametrize( + "latent_t, prefix_pixel_frames, expected_frame_flags", + [ + (1, 1, [1]), # the reference image: its single frame is given + ( + 2, + 1, + [1, 0], + ), # one fed-back pixel frame conditions the first latent frame only + (3, 5, [1, 1, 0]), # 5 = 1 + 4 pixel frames -> two latent frames + (3, 0, [0, 0, 0]), # clip 0: nothing fed back from a previous clip + ( + 3, + 9, + [1, 1, 1], + ), # the reference video: every one of its 9 pixel frames is given + ], +) +def test_make_conditioning_mask_prefix_in_pixel_frames_maps_to_latent_frames( + latent_t, prefix_pixel_frames, expected_frame_flags +): + mask = make_conditioning_mask(latent_t, 1, 2, prefix_pixel_frames, device="cpu") + assert tuple(mask.shape) == (4, latent_t, 1, 2) + assert mask.dtype == torch.float32 and mask.device.type == "cpu" + for channel in range(4): + assert mask[channel, :, 0, 0].tolist() == expected_frame_flags, channel + # Spatially constant. + assert torch.equal(mask, mask[:, :, :1, :1].expand_as(mask)) + + +def test_make_conditioning_mask_is_channel_major(): + # Flat golden pins the reshape/transpose exactly: [4, latent_t] channel-major. + mask = make_conditioning_mask(2, 1, 1, 1, device="cpu") + assert mask.flatten().tolist() == [1, 0, 1, 0, 1, 0, 1, 0] + + +# letterbox_resize / resize_by_area / LetterboxInfo.crop: how the request ``size`` is applied +# to the reference image and how the bars come back off every output frame. + + +@pytest.mark.parametrize( + "source_hw, pad_axis, content_hw", + [((40, 80), "height", (32, 64)), ((80, 40), "width", (64, 32))], +) +def test_letterbox_crop_round_trips_the_content_region(source_hw, pad_axis, content_hw): + image = np.full((*source_hw, 3), 200, np.uint8) + canvas, info = letterbox_resize(image, height=64, width=64) + assert canvas.shape == (64, 64, 3) and canvas.dtype == np.uint8 + assert info.pad_axis == pad_axis + content = info.crop(canvas[None]) + assert content.shape == (1, *content_hw, 3) + # Exactly the content pixels survive the crop; the bars are the only other pixels. + assert (content == 200).all() + assert int(canvas.sum()) == int(content.sum()) + + +@pytest.mark.parametrize( + "source_hw, expected_wh", + [ + ((1920, 1080), (528, 944)), # 9:16 reference under the 640x800 default budget + ((1080, 1920), (944, 528)), + ((1000, 1000), (704, 704)), # 1:1 + ((500, 400), (640, 800)), # the budget's own 4:5 aspect fills it exactly + ], +) +def test_resize_by_area_treats_the_request_size_as_a_pixel_area_budget( + source_hw, expected_wh +): + # The /v1/videos ``size`` and the cookbook document 640x800 -> 528x944 (9:16) and + # 704x704 (1:1): the request fixes the pixel area, the reference image fixes the aspect + # ratio, and both sides are floored to a multiple of 16. + image = np.zeros((*source_hw, 3), np.uint8) + out, _ = resize_by_area(image, 640 * 800, divisor=16) + assert (out.shape[1], out.shape[0]) == expected_wh + assert out.shape[0] * out.shape[1] <= 640 * 800 + assert out.shape[0] % 16 == 0 and out.shape[1] % 16 == 0 + + +# build_clip_conditioning: per-clip tensors from the request state with stand-in encoders. + + +def _request_state_for_clip_tests(frames, schedule, clip_len): + height, width = frames[0].shape[:2] + latent_h, latent_w = height // 8, width // 8 + inputs = _WanAnimate2Inputs( + reference_image_path="ref.png", + reference_video_path="drv.mp4", + prompt="p", + prompt_ref="r", + negative_prompt="", + width=width, + height=height, + clip_len=clip_len, + num_frames_conditioning=1, + fps=16, + seed=0, + num_inference_steps=2, + guidance_scale=1.0, + enable_audio=False, + ) + return WanAnimate2RequestState( + device=torch.device("cpu"), + sp_size=1, + generator=torch.Generator().manual_seed(0), + inputs=inputs, + letterbox_info=LetterboxInfo(pad_axis="width", padding=0, content_len=width), + reference_image_height=height, + reference_image_width=width, + latent_h=latent_h, + latent_w=latent_w, + reference_image_condition=torch.ones( + 20, 1, latent_h, latent_w, dtype=torch.bfloat16 + ), + reference_image_embeddings=torch.zeros(1, 257, 1280, dtype=torch.bfloat16), + prompt_embeddings=torch.zeros(1, 4096, dtype=torch.bfloat16), + negative_prompt_embeddings=torch.zeros(1, 4096, dtype=torch.bfloat16), + prompt_ref_embeddings=torch.zeros(1, 4096, dtype=torch.bfloat16), + reference_video_frames=frames, + schedule=schedule, + num_reference_video_frames=len(frames), + full_clip_grid_sizes=generation_token_grid(clip_len, latent_h, latent_w), + ) + + +def test_build_clip_conditioning_shapes_and_prefix_mask_contract(): + """Clip 0 has no fed-back frames and clip k has ``num_frames_conditioning`` of them; the + mask channels of ``generation_condition`` must say so frame by frame, after the + reference-image frame, or the DiT is told the wrong pixels are given.""" + clip_len = 9 # 4k+1 -> 3 latent frames of its own, 4 with the reference image + frames = _frames(17, height=32, width=48) # two full clips sharing one frame + schedule = build_schedule(frames, clip_len, 1) + assert [clip.num_frames for clip in schedule] == [clip_len, clip_len] + state = _request_state_for_clip_tests(frames, schedule, clip_len) + + def vae_encoder(videos): + return [ + torch.zeros(16, (v.shape[1] - 1) // 4 + 1, v.shape[2] // 8, v.shape[3] // 8) + for v in videos + ] + + def image_embedder(videos): + return torch.zeros(len(videos), 257, 1280) + + clip_0 = build_clip_conditioning( + state, schedule[0], None, vae_encoder=vae_encoder, image_embedder=image_embedder + ) + previous_frames = torch.zeros(3, 1, 32, 48) + clip_1 = build_clip_conditioning( + state, + schedule[1], + previous_frames, + vae_encoder=vae_encoder, + image_embedder=image_embedder, + ) + + for clip in (clip_0, clip_1): + # [20, latent_t, latent_h, latent_w]: 4 mask + 16 latent channels, latent_t = 1 + 3. + assert tuple(clip.generation_condition.shape) == (20, 4, 4, 6) + assert clip.generation_condition.dtype == torch.bfloat16 + assert clip.generation_video_grid_sizes == (4, 2, 3) + assert tuple(clip.reference_video_condition.shape) == (20, 3, 4, 6) + assert tuple(clip.reference_video_latents.shape) == (16, 3, 4, 6) + assert clip.reference_video_grid_sizes == (3, 2, 3) + assert clip.full_clip_grid_sizes == (4, 2, 3) + assert tuple(clip.reference_video_frame_0_image_embeddings.shape) == ( + 1, + 257, + 1280, + ) + assert tuple(clip.init_noise.shape) == (16, 4, 4, 6) + # Every reference-video frame is given. + assert bool((clip.reference_video_condition[:4] == 1).all()) + # Frame 0 is the reference image, given for every clip. + assert bool((clip.generation_condition[:4, 0] == 1).all()) + # Clip 0: none of its own frames is given; clip 1: the one fed-back frame is. + assert clip_0.generation_condition[:4, 1:].abs().sum() == 0 + assert bool((clip_1.generation_condition[:4, 1] == 1).all()) + assert clip_1.generation_condition[:4, 2:].abs().sum() == 0 + + # A later clip without the previous clip's frames is a caller bug, not a silent clip 0. + with pytest.raises(ValueError, match="prev_clip_conditioning_frames"): + build_clip_conditioning( + state, + schedule[1], + None, + vae_encoder=vae_encoder, + image_embedder=image_embedder, + ) + + +def test_build_clip_conditioning_uses_the_stage_drawn_noise_for_clip_0_only(): + # The before-denoising stage draws clip 0's noise into batch.latents; the clip loop must + # consume that draw instead of drawing again, and later clips must keep drawing. + clip_len = 9 + frames = _frames(17, height=32, width=48) + schedule = build_schedule(frames, clip_len, 1) + state = _request_state_for_clip_tests(frames, schedule, clip_len) + stage_drawn = torch.full((16, 4, 4, 6), 7.0) + state.clip_0_init_noise = stage_drawn + + def vae_encoder(videos): + return [ + torch.zeros(16, (v.shape[1] - 1) // 4 + 1, v.shape[2] // 8, v.shape[3] // 8) + for v in videos + ] + + def image_embedder(videos): + return torch.zeros(len(videos), 257, 1280) + + clip_0 = build_clip_conditioning( + state, schedule[0], None, vae_encoder=vae_encoder, image_embedder=image_embedder + ) + clip_1 = build_clip_conditioning( + state, + schedule[1], + torch.zeros(3, 1, 32, 48), + vae_encoder=vae_encoder, + image_embedder=image_embedder, + ) + assert clip_0.init_noise is stage_drawn + assert clip_1.init_noise is not stage_drawn + assert tuple(clip_1.init_noise.shape) == (16, 4, 4, 6) + assert not torch.equal(clip_1.init_noise, stage_drawn) + + +# Before-denoising stage: _WanAnimate2Inputs.from_request reads the SamplingParam-backed fields + + +def test_extract_inputs_reads_request_fields_through_the_sampling_params(): + """prompt_ref must reach the reference branch as its own text, separate from the character + caption that feeds the generation branch.""" + params = Wan_Animate_2_14B_SamplingParam( + prompt="p", + image_path="ref.png", + video_path="drv.mp4", + width=640, + height=800, + clip_len=45, + fps=24, + seed=7, + ) + inputs = _WanAnimate2Inputs.from_request(Req(sampling_params=params)) + + # Non-default values prove the fields are read off the request, not the defaults. + assert (inputs.clip_len, inputs.fps, inputs.seed) == (45, 24, 7) + assert (inputs.width, inputs.height) == (640, 800) + assert inputs.num_frames_conditioning == 1 + assert (inputs.reference_image_path, inputs.reference_video_path) == ( + "ref.png", + "drv.mp4", + ) + assert inputs.prompt == "p" + assert inputs.prompt_ref == params.prompt_ref + assert inputs.prompt_ref != inputs.prompt + assert inputs.negative_prompt == params.negative_prompt + assert inputs.enable_audio is True + + +# Carrying the audio track is a per-request knob, on by default like the official pipeline + + +def test_enable_audio_is_on_by_default_and_settable_over_the_video_api(): + assert Wan_Animate_2_14B_SamplingParam().enable_audio is True + assert "enable_audio" not in VideoGenerationsRequest.model_fields + assert ( + "enable_audio" in Wan_Animate_2_14B_SamplingParam.video_request_extra_fields() + ) + assert not hasattr(Wan_Animate_2_14B_Config(), "enable_audio") + + +def _audio_stage(monkeypatch, calls: list[str], *, no_audio_track: bool = False): + """A before-denoising stage whose audio extractor records its calls instead of decoding + the file and reports either a silent reference video or a decode failure.""" + import sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising as mod + + def fake_extract(path, *, log_info, log_warning, log_no_audio_track): + calls.append(path) + if no_audio_track: + log_no_audio_track( + "Wan-Animate-2: reference video %s has no audio track", path + ) + else: + log_warning("Wan-Animate-2: cannot decode the audio track of %s", path) + return None, None + + monkeypatch.setattr(mod, "extract_reference_video_audio", fake_extract) + stage = WanAnimate2BeforeDenoisingStage.__new__(WanAnimate2BeforeDenoisingStage) + stage._silent_reference_logged = False + stage._audio_failure_logged = False + stage.infos = [] + stage.warnings = [] + stage.debugs = [] + stage.log_info = lambda msg, *args: stage.infos.append(msg % args) + stage.log_warning = lambda msg, *args: stage.warnings.append(msg % args) + stage.log_debug = lambda msg, *args: stage.debugs.append(msg % args) + return stage + + +def _inputs_with_audio(enable_audio: bool): + params = Wan_Animate_2_14B_SamplingParam( + prompt="p", + image_path="ref.png", + video_path="drv.mp4", + width=640, + height=800, + enable_audio=enable_audio, + ) + return _WanAnimate2Inputs.from_request(Req(sampling_params=params)) + + +def test_audio_extraction_is_skipped_when_the_request_turns_it_off(monkeypatch): + calls: list[str] = [] + stage = _audio_stage(monkeypatch, calls) + + assert stage._extract_audio(_inputs_with_audio(False)) == (None, None) + assert calls == [] + + stage._extract_audio(_inputs_with_audio(True)) + assert calls == ["drv.mp4"] + + +def test_silent_reference_video_logs_info_once_then_debug(monkeypatch): + stage = _audio_stage(monkeypatch, [], no_audio_track=True) + inputs = _inputs_with_audio(True) + + stage._extract_audio(inputs) + stage._extract_audio(inputs) + + assert stage.warnings == [] + assert len(stage.infos) == 1 + assert "no audio track" in stage.infos[0] and "enable_audio=false" in stage.infos[0] + assert stage.debugs == ["Wan-Animate-2: reference video drv.mp4 has no audio track"] + + +def test_audio_failures_warn_once_per_process_then_go_to_debug(monkeypatch): + stage = _audio_stage(monkeypatch, []) + inputs = _inputs_with_audio(True) + + stage._extract_audio(inputs) + stage._extract_audio(inputs) + + assert len(stage.warnings) == 1 + assert "drv.mp4" in stage.warnings[0] + assert "enable_audio=false" in stage.warnings[0] + assert "debug level" in stage.warnings[0] + assert stage.debugs == ["Wan-Animate-2: cannot decode the audio track of drv.mp4"] + # A silent reference afterwards still gets its own first-time info line. + assert stage.infos == [] + + +# extract_reference_video_audio: the shared in-process reader decodes the track at its native +# rate; a video without an audio track is reported through the dedicated logger, any reader +# failure through one generic warning carrying the reader's message. + + +def _extract_with_fake_reader(monkeypatch, *, sample_rate, samples=None, error=None): + """Run the extractor with the stream probe answering ``sample_rate`` and the shared reader + returning ``samples`` or raising ``error``; return ``(result, logs, reader_calls)``.""" + import sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio as mod + + reader_calls: list[tuple] = [] + + def fake_decode(path, *, target_sr, mono): + reader_calls.append((path, target_sr, mono)) + if error is not None: + raise error + return samples + + monkeypatch.setattr(mod, "_audio_track_sample_rate", lambda path: sample_rate) + monkeypatch.setattr(mod, "decode_audio_container", fake_decode) + logs = {"info": [], "warning": [], "no_audio_track": []} + result = mod.extract_reference_video_audio( + "drv.mp4", + log_info=lambda msg, *args: logs["info"].append(msg % args), + log_warning=lambda msg, *args: logs["warning"].append(msg % args), + log_no_audio_track=lambda msg, *args: logs["no_audio_track"].append(msg % args), + ) + return result, logs, reader_calls + + +def test_extracted_audio_is_batched_channels_first_at_the_native_rate(monkeypatch): + # The reader hands back [L, C] at the requested rate. + samples = np.linspace(-1.0, 1.0, 2 * 480, dtype=np.float32).reshape(480, 2) + (audio, sample_rate), logs, reader_calls = _extract_with_fake_reader( + monkeypatch, sample_rate=48000, samples=samples + ) + assert reader_calls == [("drv.mp4", 48000, False)] + assert sample_rate == 48000 + assert audio.shape == (1, 2, 480) and audio.dtype == torch.float32 + assert audio.is_contiguous() + torch.testing.assert_close(audio[0], torch.from_numpy(samples).T) + assert logs["warning"] == [] and logs["no_audio_track"] == [] + assert logs["info"] == [ + "Wan-Animate-2: extracted reference-video audio (1, 2, 480) @ 48000 Hz." + ] + + +def test_missing_audio_track_goes_to_the_no_audio_track_logger(monkeypatch): + result, logs, reader_calls = _extract_with_fake_reader( + monkeypatch, sample_rate=None + ) + assert result == (None, None) + assert reader_calls == [] + assert logs["warning"] == [] and logs["info"] == [] + assert logs["no_audio_track"] == [ + "Wan-Animate-2: reference video drv.mp4 has no audio track; the output is silent." + ] + + +def test_reader_failures_warn_with_the_reader_message(monkeypatch): + result, logs, _ = _extract_with_fake_reader( + monkeypatch, + sample_rate=44100, + error=ValueError("no decodable audio stream was found in the media container"), + ) + assert result == (None, None) + assert logs["info"] == [] and logs["no_audio_track"] == [] + assert logs["warning"] == [ + "Wan-Animate-2: cannot decode the audio track of reference video drv.mp4 (no " + "decodable audio stream was found in the media container); the output is silent." + ] + + +def test_missing_reader_dependency_warns_instead_of_raising(monkeypatch): + import sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio as mod + + def probe(path): + raise ImportError("No module named 'av'") + + monkeypatch.setattr(mod, "_audio_track_sample_rate", probe) + warnings: list[str] = [] + result = mod.extract_reference_video_audio( + "drv.mp4", + log_info=lambda msg, *args: pytest.fail("no info line expected"), + log_warning=lambda msg, *args: warnings.append(msg % args), + ) + assert result == (None, None) + assert warnings == [ + "Wan-Animate-2: cannot decode the audio track of reference video drv.mp4 (No module " + "named 'av'); the output is silent." + ] + + +@pytest.mark.parametrize( + "overrides, message", + [ + ({"image_path": ["a.png", "b.png"]}, "exactly one 'image_path'"), + ({"video_path": []}, "exactly one 'video_path'"), + ({"seed": [1, 2]}, "exactly one 'seed'"), + ], +) +def test_extract_inputs_rejects_batched_single_value_fields(overrides, message): + # One video per request: a batched reference image, driving video or seed list must be + # refused here, not fail inside the stage with an index error or run the wrong seed. + params = Wan_Animate_2_14B_SamplingParam( + prompt="p", image_path="ref.png", video_path="drv.mp4" + ) + batch = Req(sampling_params=params, **overrides) + with pytest.raises(ValueError, match=message): + _WanAnimate2Inputs.from_request(batch) + + +# Scheduler: loaded from the checkpoint's scheduler/scheduler_config.json through the shared +# SchedulerLoader like the other components. The stages pass the official sigma grid to +# set_timesteps explicitly, so the loaded configuration must denoise exactly like a +# DPMSolverMultistepScheduler built from the flow-matching settings alone. + +# Wan2.2-Animate-2-14B-Diffusers/scheduler/scheduler_config.json without _class_name, which +# the loader pops before constructing the class it names. +_CHECKPOINT_SCHEDULER_CONFIG = { + "_diffusers_version": "0.40.0.dev0", + "algorithm_type": "dpmsolver++", + "beta_end": 0.02, + "beta_schedule": "linear", + "beta_start": 0.0001, + "disable_corrector": [], + "dynamic_thresholding_ratio": 0.995, + "euler_at_final": False, + "final_sigmas_type": "zero", + "flow_shift": 5.0, + "lambda_min_clipped": -math.inf, + "lower_order_final": True, + "num_train_timesteps": 1000, + "predict_x0": True, + "prediction_type": "flow_prediction", + "rescale_betas_zero_snr": False, + "sample_max_value": 1.0, + "solver_order": 2, + "solver_p": None, + "solver_type": "midpoint", + "steps_offset": 0, + "thresholding": False, + "time_shift_type": "exponential", + "timestep_spacing": "linspace", + "trained_betas": None, + "use_beta_sigmas": False, + "use_dynamic_shifting": False, + "use_exponential_sigmas": False, + "use_flow_sigmas": True, + "use_karras_sigmas": False, + "use_lu_lambdas": False, + "variance_type": None, +} + +_FLOW_MATCHING_SCHEDULER_KWARGS = { + "num_train_timesteps": 1000, + "prediction_type": "flow_prediction", + "use_flow_sigmas": True, + "flow_shift": 5.0, +} + + +def test_scheduler_is_a_loaded_component(): + assert "scheduler" in WanAnimate2Pipeline._required_config_modules + assert len(WanAnimate2Pipeline._required_config_modules) == 7 + # No pipeline-side construction: the base class hook is inherited unchanged. + assert "initialize_pipeline" not in vars(WanAnimate2Pipeline) + + +def test_loader_keeps_the_checkpoint_scheduler_settings(): + kwargs = _supported_init_kwargs( + DPMSolverMultistepScheduler, dict(_CHECKPOINT_SCHEDULER_CONFIG) + ) + assert kwargs["beta_schedule"] == "linear" + assert kwargs["use_flow_sigmas"] is True + assert kwargs["prediction_type"] == "flow_prediction" + assert kwargs["flow_shift"] == 5.0 + scheduler = DPMSolverMultistepScheduler(**kwargs) + config = scheduler._inner.config + assert config.algorithm_type == "dpmsolver++" + assert config.solver_order == 2 + assert config.beta_schedule == "linear" + assert config.use_flow_sigmas is True + assert config.prediction_type == "flow_prediction" + assert config.flow_shift == 5.0 + + +@pytest.mark.parametrize("num_inference_steps", [4, 40]) +def test_checkpoint_scheduler_denoises_like_the_flow_matching_settings( + num_inference_steps, +): + # beta_schedule is the only setting that differs between the two configurations + # (linear vs the wrapper's scaled_linear default); it only shapes the beta-derived + # tables, which the explicit sigma grid path never reads. + from_checkpoint = DPMSolverMultistepScheduler(**_CHECKPOINT_SCHEDULER_CONFIG) + from_settings = DPMSolverMultistepScheduler(**_FLOW_MATCHING_SCHEDULER_KWARGS) + sigmas = get_sampling_sigmas(num_inference_steps, 5.0) + from_checkpoint.set_timesteps(sigmas=sigmas) + from_settings.set_timesteps(sigmas=sigmas) + assert torch.equal(from_checkpoint.timesteps, from_settings.timesteps) + assert torch.equal(from_checkpoint.sigmas, from_settings.sigmas) + assert len(from_checkpoint.timesteps) == num_inference_steps + + generator = torch.Generator().manual_seed(0) + sample = torch.randn(1, 16, 2, 4, 4, generator=generator) + model_output = torch.randn(1, 16, 2, 4, 4, generator=generator) + t = from_checkpoint.timesteps[0] + out_checkpoint = from_checkpoint.step(model_output, t, sample).prev_sample + out_settings = from_settings.step(model_output, t, sample).prev_sample + assert torch.equal(out_checkpoint, out_settings) + + +# Official sampler sigma grid + + +def test_official_sampler_shift_and_sigma_grid(): + # Both literals come from the official sampler (sample_shift 5.0 and its + # shift * s / (1 + (shift - 1) * s) grid); the config's shift must feed that grid. + config = Wan_Animate_2_14B_Config() + assert config.flow_shift == 5.0 + sigmas = get_sampling_sigmas(40, config.flow_shift) + assert len(sigmas) == 40 + assert sigmas[0] == 1.0 + np.testing.assert_array_equal( + sigmas[:3], [1.0, 0.9948979591836734, 0.9895833333333334] + ) + + +# WanAnimate2VaeAdapter wraps the sglang AutoencoderKLWan, whose encode returns the +# DiagonalGaussianDistribution itself and whose decode returns a plain tensor (no diffusers +# AutoencoderKLOutput / DecoderOutput wrappers). +def test_vae_adapter_uses_the_sglang_vae_return_types(): + class _StubWanVae(torch.nn.Module): + latents_mean = [0.5, -0.5] + latents_std = [2.0, 4.0] + use_parallel_decode = True + tiled = False + + def enable_tiling(self): + self.tiled = True + + def encode(self, x): + # mean = x's per-channel mean, logvar = 0, in the [B, 2C, T, H, W] layout. + mean = x.mean(dim=(2, 3, 4), keepdim=True).expand(-1, -1, 1, 1, 1) + return DiagonalGaussianDistribution( + torch.cat([mean, torch.zeros_like(mean)], dim=1) + ) + + def decode(self, z): + return z * 3.0 # a plain tensor, as the in-tree VAE returns + + adapter = WanAnimate2VaeAdapter(_StubWanVae()) + + pixels = torch.zeros(2, 1, 4, 4) + pixels[0] += 2.5 # channel 0 mean 2.5, channel 1 mean 0.0 + (latent,) = adapter.encode([pixels]) + # (mode - mean) / std per channel: (2.5 - 0.5) / 2 = 1.0, (0.0 + 0.5) / 4 = 0.125 + torch.testing.assert_close(latent.flatten(), torch.tensor([1.0, 0.125])) + assert latent.dtype == torch.float32 + + decoded = adapter.decode([latent]) + # denormalize (z * std + mean) = [2.5, 0.0], then the stub's x3, clamped to [-1, 1] + torch.testing.assert_close(decoded.flatten(), torch.tensor([1.0, 0.0])) + assert decoded.shape[0] == 1 and decoded.dtype == torch.float32 + + args = SimpleNamespace( + pipeline_config=Wan_Animate_2_14B_Config(), + disable_autocast=False, + enable_torch_compile=False, + ) + args.pipeline_config.vae_tiling = True + stage = DecodingStage(adapter) + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.decoding.get_local_torch_device", + return_value=torch.device("cpu"), + ): + raw = stage.decode_raw(latent.unsqueeze(0), args, vae_dtype=torch.float32) + normalized = stage.decode(latent.unsqueeze(0), args, vae_dtype=torch.float32) + torch.testing.assert_close(raw, decoded, rtol=0, atol=0) + torch.testing.assert_close(normalized, (decoded / 2 + 0.5).clamp(0, 1)) + assert adapter.vae.tiled and adapter.use_parallel_decode + torch.testing.assert_close( + adapter.decode([latent, latent]), decoded.expand(2, -1, -1, -1, -1) + ) + + +# Deployment topology + + +def test_disaggregated_roles_are_rejected_at_argument_validation(): + """The pipeline's inter-stage state lives in batch.extra as GPU tensors, which the + disaggregation transport cannot carry; a split-role launch must fail before any weights + load instead of at the first request.""" + for role in (RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER): + args = SimpleNamespace( + pipeline_config=Wan_Animate_2_14B_Config(), disagg_role=role + ) + with pytest.raises(ValueError, match="only supports monolithic"): + ServerArgs._validate_disagg_capability(args) + ServerArgs._validate_disagg_capability( + SimpleNamespace( + pipeline_config=Wan_Animate_2_14B_Config(), disagg_role=RoleType.MONOLITHIC + ) + ) + + +# Synthetic server warmup. The generic warmup builder supplies only ``image_path``; a +# Wan-Animate-2 warmup request must also carry a decodable driving video that the clip +# schedule accepts as one clip, or ``--warmup-mode server`` (forced on by +# ``--enable-torch-compile``) can never finish. Requests are built exactly as +# ``run_async_client_warmup`` builds them and run through the before-denoising stage with +# stand-in encoders. + +_BEFORE_DENOISING = ( + "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages" + ".wan_animate_2.before_denoising" +) + + +def _server_args(input_save_path: str) -> SimpleNamespace: + return SimpleNamespace( + pipeline_class_name="WanAnimate2Pipeline", + pipeline_config=Wan_Animate_2_14B_Config(), + component_precisions={}, + enable_layerwise_nvtx_marker=False, + comfyui_mode=False, + warmup_steps=1, + warmup_num_frames=None, + warmup_resolutions=None, + enable_torch_compile=False, + enable_breakable_cuda_graph=False, + enable_cfg_parallel=False, + num_gpus=1, + backend="sglang", + # anything but "auto": keeps the auto-residency full-shape probe out of the build + performance_mode="balanced", + input_save_path=input_save_path, + is_arg_explicitly_set=lambda name: False, + ) + + +def _build_server_warmup_reqs(server_args: SimpleNamespace) -> list[Req]: + warmup_image = prepare_warmup_image_path(server_args) + with patch( + "sglang.multimodal_gen.runtime.warmup_request_builder.get_model_sampling_defaults", + return_value=Wan_Animate_2_14B_SamplingParam(), + ): + return build_warmup_reqs( + server_args, + warmup_resolutions=None, + warmup_input_path=warmup_image, + server_based_warmup=True, + ) + + +def _single_clip_schedule(inputs: _WanAnimate2Inputs) -> list[int]: + """Per-clip frame counts the stage would schedule for the request's driving video.""" + frames = read_reference_video_frames(inputs.reference_video_path, inputs.fps) + assert frames.ndim == 4 and frames.shape[-1] == 3 and len(frames) > 0 + padded = zigzag_padding(list(frames), get_padding_len(len(frames), inputs.clip_len)) + return [ + clip.num_frames + for clip in build_schedule( + padded, inputs.clip_len, inputs.num_frames_conditioning + ) + ] + + +def test_reference_video_url_uses_native_media_loading(tmp_path): + path = tmp_path / "reference.mp4" + # keep the tiny fixture's width aligned for decord's RGB frame buffer + write_reference_video(str(path), np.full((5, 32, 64, 3), 128, dtype=np.uint8), 16) + expected = read_reference_video_frames(str(path), 8) + assert expected.shape == (2, 32, 64, 3) + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages." + "wan_animate_2.preprocess.get_video_bytes", + return_value=path.read_bytes(), + ) as download: + actual = read_reference_video_frames("https://example.com/reference.mp4", 8) + download.assert_called_once_with("https://example.com/reference.mp4") + np.testing.assert_array_equal(actual, expected) + + +class _StubVae: + """encode: [3, T, H, W] -> [16, (T-1)//4+1, H//8, W//8] fp32 zeros; the VAE module itself + is only handed to the residency manager.""" + + def __init__(self) -> None: + self.vae = torch.nn.Identity() + + def encode(self, videos: list[torch.Tensor]) -> list[torch.Tensor]: + return [ + torch.zeros(16, (v.shape[1] - 1) // 4 + 1, v.shape[2] // 8, v.shape[3] // 8) + for v in videos + ] + + +class _StubImageEncoder: + def __init__(self) -> None: + self.model = torch.nn.Identity() + + def visual(self, videos: list[torch.Tensor]) -> torch.Tensor: + return torch.zeros(len(videos), 257, 1280) + + +def _before_denoising_stage(pipeline_config) -> WanAnimate2BeforeDenoisingStage: + stage = WanAnimate2BeforeDenoisingStage( + vae=_StubVae(), + image_encoder=_StubImageEncoder(), + text_encoder=torch.nn.Identity(), + tokenizer=object(), + pipeline_config=pipeline_config, + scheduler=DPMSolverMultistepScheduler( + num_train_timesteps=1000, + prediction_type="flow_prediction", + use_flow_sigmas=True, + flow_shift=pipeline_config.flow_shift, + ), + ) + stage.encode_text = lambda text, *args, **kwargs: ( + [torch.full((max(1, len(text)), 4096), 0.5)], + [], + ) + return stage + + +def test_server_warmup_request_denoises_one_clip_of_a_synthetic_driving_video(tmp_path): + server_args = _server_args(str(tmp_path)) + reqs = _build_server_warmup_reqs(server_args) + assert len(reqs) == 1 + + inputs = _WanAnimate2Inputs.from_request(reqs[0]) + assert os.path.isfile(inputs.reference_image_path) + assert os.path.isfile(inputs.reference_video_path) + assert os.path.dirname(inputs.reference_video_path) == str(tmp_path) + assert inputs.num_inference_steps == SERVER_WARMUP_VIDEO_STEPS + # Warm up at the default production clip length so the compiled attention is + # specialized for the geometry real requests use (the builder alone asks for 17 frames). + assert inputs.clip_len == Wan_Animate_2_14B_SamplingParam().clip_len + assert _single_clip_schedule(inputs) == [inputs.clip_len] + # The synthetic driving video is silent, so warmup must not decode audio for it. + assert inputs.enable_audio is False + + +def test_warmup_requests_with_different_clip_lengths_keep_separate_driving_videos( + tmp_path, +): + # All warmup requests are built before the first one runs, so the bounded request and + # the full-shape probe must not overwrite each other's video. + server_args = _server_args(str(tmp_path)) + warmup_image = prepare_warmup_image_path(server_args) + inputs = [] + for num_frames in (17, 45): + params = Wan_Animate_2_14B_SamplingParam() + req = Req( + sampling_params=params, + prompt="warmup", + image_path=[warmup_image], + width=640, + height=800, + num_frames=num_frames, + ) + params.prepare_synthetic_warmup_request_for_queue(req, server_args) + inputs.append(_WanAnimate2Inputs.from_request(req)) + + default_len, longer = inputs + assert default_len.reference_video_path != longer.reference_video_path + assert default_len.clip_len == Wan_Animate_2_14B_SamplingParam().clip_len == 37 + assert longer.clip_len == 45 + assert _single_clip_schedule(default_len) == [default_len.clip_len] + assert _single_clip_schedule(longer) == [longer.clip_len] + + +_SHARED_VIDEO_WRITER = ( + "sglang.multimodal_gen.runtime.entrypoints.utils.post_process_sample" +) + + +def test_write_reference_video_goes_through_the_shared_video_writer(tmp_path): + # The synthetic warmup video is encoded by the writer request outputs go through, with + # the frames handed over unchanged, so it decodes wherever request outputs decode. + frames = np.full((9, 64, 64, 3), 200, dtype=np.uint8) + path = os.path.join(str(tmp_path), "warmup_driving_video_9f_30fps.mp4") + + with patch(_SHARED_VIDEO_WRITER) as writer: + write_reference_video(path, frames, fps=30) + + writer.assert_called_once() + (sample, data_type, fps), kwargs = writer.call_args + assert sample is frames + assert data_type == DataType.VIDEO + assert fps == 30 + assert kwargs == {"save_file_path": path} + + +def test_write_reference_video_names_the_shared_writer_when_it_fails(tmp_path): + path = os.path.join(str(tmp_path), "warmup_driving_video_9f_30fps.mp4") + frames = np.zeros((9, 64, 64, 3), dtype=np.uint8) + + with ( + patch(_SHARED_VIDEO_WRITER, side_effect=OSError("no video backend")), + pytest.raises(RuntimeError, match="shared video writer") as excinfo, + ): + write_reference_video(path, frames, fps=30) + + assert path in str(excinfo.value) + assert "no video backend" in str(excinfo.value) + assert isinstance(excinfo.value.__cause__, OSError) + + +@pytest.mark.parametrize("clip_len", [17, 37]) +def test_before_denoising_stage_populates_the_standard_denoising_inputs( + tmp_path, clip_len +): + """After the before-denoising stage the Req must carry the fields the shared denoising + contract requires, consistent with what the clip loop will use: clip 0's initial noise + (the seed's first draw, which the loop consumes instead of drawing again) and the + official sigma grid.""" + server_args = _server_args(str(tmp_path)) + (req,) = _build_server_warmup_reqs(server_args) + req.clip_len = clip_len + set_global_server_args(server_args) + stage = _before_denoising_stage(server_args.pipeline_config) + + with ( + patch(f"{_BEFORE_DENOISING}.get_sp_world_size", return_value=1), + patch( + f"{_BEFORE_DENOISING}.get_local_torch_device", + return_value=torch.device("cpu"), + ), + ): + batch = stage(req, server_args) + + request_state = request_state_from_batch(batch) + assert all( + clip.frame_start_index + clip.num_frames_conditioning_for_clip + < request_state.num_reference_video_frames + for clip in request_state.schedule + ) + assert batch.generator is request_state.generator + assert batch.prompt_embeds == [request_state.prompt_embeddings] + assert batch.negative_prompt_embeds == [request_state.negative_prompt_embeddings] + assert batch.image_embeds == [request_state.reference_image_embeddings] + assert len(batch.sigmas) == batch.num_inference_steps == len(batch.timesteps) + assert batch.sigmas[0] == 1.0 + assert torch.equal( + batch.timesteps, torch.tensor([int(s * 1000) for s in batch.sigmas]) + ) + + clip_0 = build_clip_conditioning( + request_state, + request_state.schedule[0], + None, + vae_encoder=stage.vae_encoder, + image_embedder=stage.image_embedder, + ) + assert torch.equal(batch.latents, clip_0.init_noise) + assert tuple(batch.raw_latent_shape) == tuple(clip_0.init_noise.shape) + + # One-draw-ahead invariant: clip 0 is the seed's first draw of its shape, and the + # request generator now stands exactly one draw further, so the next clip gets the + # seed's second draw, as it did when the loop drew clip 0 itself. + reference = torch.Generator(device=request_state.device).manual_seed( + request_state.inputs.seed + ) + shape = tuple(batch.latents.shape) + first = torch.randn(*shape, device=request_state.device, generator=reference) + second = torch.randn(*shape, device=request_state.device, generator=reference) + assert torch.equal(first, batch.latents) + next_clip_noise = torch.randn( + *shape, device=request_state.device, generator=request_state.generator + ) + assert torch.equal(next_clip_noise, second) + assert not torch.equal(next_clip_noise, batch.latents) + + denoising_verification = WanAnimate2DenoisingStage.verify_input( + None, batch, server_args + ) + assert denoising_verification.is_valid(), ( + denoising_verification.get_failure_summary() + ) + # A request that skipped the before-denoising stage must not pass the same check. + bare = WanAnimate2DenoisingStage.verify_input( + None, Req(sampling_params=Wan_Animate_2_14B_SamplingParam()), server_args + ) + assert not bare.is_valid() diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py new file mode 100644 index 0000000000..0bef27689d --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py @@ -0,0 +1,799 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for the Wan-Animate-2 block mask, shape math and per-clip reference K/V ownership. + +Single-process and CPU only, in the style of ``test_sp_shard.py`` / ``test_usp_ring_tail_pad.py``: +``_get_block_mask_from_layout`` is called unbound with a stub ``self`` and patched module globals; the +shape helpers are called directly. The parallelism rejections live in ``test_wan_animate_2_config.py``. +""" + +from collections import OrderedDict +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + maybe_init_distributed_environment_and_model_parallel, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.models.dits import wan_animate_2_block +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import ( + WanAnimate2Transformer3DModel, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import ( + WanAnimate2TransformerBlock, + _apply_rope_interleaved, + _InContextLayout, + _make_score_mod, + _sp_scatter_out, +) +from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import ( + WanAnimate2ClipConditioning, + WanAnimate2ReferenceKV, +) +from sglang.multimodal_gen.runtime.models.dits.wanvideo import ( + WanTransformer3DModel, + WanTransformerBlock, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import ( + WanAnimate2DenoisingStage, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( + ensure_distributed_env_defaults, +) + +_DIT = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2" +_BLOCK = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block" +_DENOISING = ( + "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages" + ".wan_animate_2.denoising" +) + + +# A short last clip against the full-length clip the mask is built for: this clip is +# (2, 1, 2) = 2 frames x 2 tokens, the reference video (1, 1, 2), the full clip (4, 2, 2) = +# 4 frames x 4 tokens. Every mask-side field (max_*) differs from the clip's own, so a mask +# built from the clip's grid, or attention padding that disagrees with the mask, fails below. +_SHORT_CLIP_LAYOUT = _InContextLayout((2, 1, 2), (1, 1, 2), (4, 2, 2)) + + +def _mask_probe(mask_fn): + def valid(q_idx, kv_idx): + out = mask_fn( + torch.tensor(0), + torch.tensor(0), + torch.tensor(q_idx), + torch.tensor(kv_idx), + ) + return bool(out.item()) + + return valid + + +def test_block_mask_head_agnostic_and_masks_correctly(): + captured = {} + + def fake_create_block_mask(mask_fn, *, B, H, Q_LEN, KV_LEN, device, _compile): + captured.update(mask_fn=mask_fn, B=B, H=H, Q_LEN=Q_LEN, KV_LEN=KV_LEN) + return "SENTINEL_BLOCK_MASK" + + stub = SimpleNamespace(_block_masks=OrderedDict(), _device=torch.device("cpu")) + layout = _SHORT_CLIP_LAYOUT + with patch(f"{_DIT}.create_block_mask", side_effect=fake_create_block_mask) as cbm: + first = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout) + second = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout) + + # Cached: the second call never rebuilds. + assert first == "SENTINEL_BLOCK_MASK" + assert second is first + assert cbm.call_count == 1 + + # Head/batch-agnostic so SP head-sharding cannot change the mask. + assert captured["B"] is None + assert captured["H"] is None + # Sized from the full clip (16 gen + 12 reference-video tokens), each side ceil'd to 128. + assert captured["Q_LEN"] == 128 + assert captured["KV_LEN"] == 256 + + valid = _mask_probe(captured["mask_fn"]) + # Generation queries attend every generation token of the full clip, not the gen pad. + assert valid(0, 0) and valid(5, 15) + assert not valid(5, 16) + # A pad query row attends nothing. + assert not valid(16, 0) + # Frame stride is the full clip's 4 tokens per frame, not this clip's 2: query 5 is in + # generation frame 1 and attends reference-video frame 1 (keys 128..131) only. A mask + # built from the clip's own grid would put query 5 in frame 2 and query 2 in frame 1. + assert valid(5, 128) and valid(5, 131) + assert not valid(5, 132) + assert not valid(2, 128) + # Frame 0 is the reference image; it has no reference-video counterpart. + assert not valid(0, 128) + # Frame 3 attends keys 136..139; 140 is past the 12 reference-video tokens (pad). + assert valid(13, 136) and valid(13, 139) + assert not valid(13, 140) + + +@pytest.mark.parametrize("cache_kind", ["rope", "mask"]) +def test_geometry_cache_bounds_and_rebuilds(cache_kind): + cache = OrderedDict() + stub = SimpleNamespace(_device=torch.device("cpu")) + if cache_kind == "rope": + limit = 4 + stub._rope_cache = cache + stub.rotary_emb = SimpleNamespace( + forward_uncached=lambda positions: (positions.cos(), positions.sin()) + ) + + def lookup(size): + return WanAnimate2Transformer3DModel._rope_tables( + stub, (0, size, 1), (0, 2), (0, 2) + ) + + context = nullcontext() + else: + limit = 2 + stub._block_masks = cache + + def lookup(size): + layout = _InContextLayout((size, 2, 2), (size - 1, 2, 2), (size, 2, 2)) + return WanAnimate2Transformer3DModel._get_block_mask_from_layout( + stub, layout + ) + + context = patch( + f"{_DIT}.create_block_mask", + side_effect=lambda fn, **kwargs: torch.tensor( + [_mask_probe(fn)(0, i) for i in range(32)] + ), + ) + with context: + first = lookup(2) + oldest_key = next(iter(cache)) + for size in range(3, limit + 2): + lookup(size) + assert lookup(2) is first + lookup(limit + 2) + assert oldest_key in cache + for size in range(limit + 3, 2 * limit + 3): + lookup(size) + assert len(cache) == limit + assert oldest_key not in cache + rebuilt = lookup(2) + torch.testing.assert_close(rebuilt, first, rtol=0, atol=0) + + +def test_attention_pads_tokens_into_the_slots_the_block_mask_admits(): + # The mask is built once per request from the full clip while the attention pads every + # clip's tokens; a frame stride or reference-video offset that drifts between the two + # is a silent attention-pattern bug no shape check catches. + layout = _SHORT_CLIP_LAYOUT + captured = {} + + def fake_flex_attention(q, k, v, *, block_mask, score_mod): + captured.update(q=q, k=k, v=v, block_mask=block_mask) + return torch.zeros_like(q) + + batch, heads, dim = 1, 2, 8 + + def tokens(values, scale): + base = torch.tensor(values, dtype=torch.float32).view(1, len(values), 1, 1) + return (base * scale).expand(batch, len(values), heads, dim).clone() + + # Token ids 1..4 are generation frame 0 (1, 2) and frame 1 (3, 4); 1, 2 x 1000 are the + # reference-video frame. + q, k, v = ( + tokens([1, 2, 3, 4], 1.0), + tokens([1, 2, 3, 4], 10.0), + tokens([1, 2, 3, 4], 100.0), + ) + reference_k, reference_v = tokens([1, 2], 1000.0), tokens([1, 2], 10000.0) + stub = SimpleNamespace(log_scale=0.0) + with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention): + out = WanAnimate2TransformerBlock._in_context_attention( + stub, q, k, v, reference_k, reference_v, "SENTINEL_BLOCK_MASK", layout + ) + + assert captured["block_mask"] == "SENTINEL_BLOCK_MASK" + assert tuple(out.shape) == (batch, 4, heads, dim) + q_padded = captured["q"].transpose(1, 2) # back to [B, L, N, D] + k_padded = captured["k"].transpose(1, 2) + v_padded = captured["v"].transpose(1, 2) + assert tuple(q_padded.shape) == ( + batch, + layout.padded_generation_video_len, + heads, + dim, + ) + assert tuple(k_padded.shape) == (batch, layout.padded_kv_len, heads, dim) + + def slot(t, i): + return t[0, i, 0, 0].item() + + # Generation tokens sit frame by frame at the full clip's 4-token stride. + assert [slot(q_padded, i) for i in range(8)] == [1, 2, 0, 0, 3, 4, 0, 0] + assert [slot(k_padded, i) for i in range(8)] == [10, 20, 0, 0, 30, 40, 0, 0] + assert q_padded[:, 8:].abs().sum() == 0 + # Reference-video tokens start at padded_generation_video_len with the same stride. + reference_start = layout.padded_generation_video_len + assert [slot(k_padded, reference_start + i) for i in range(4)] == [1000, 2000, 0, 0] + assert [slot(v_padded, reference_start + i) for i in range(4)] == [ + 10000, + 20000, + 0, + 0, + ] + assert k_padded[:, 8:reference_start].abs().sum() == 0 + assert k_padded[:, reference_start + 4 :].abs().sum() == 0 + + # Every slot that holds a real token is one the mask built from the same layout routes: + # all gen pairs, and frame-1 queries (slots 4, 5) to the reference-video keys. + valid = _mask_probe(layout.block_mask_fn()) + gen_slots = [0, 1, 4, 5] + assert all(valid(qi, ki) for qi in gen_slots for ki in gen_slots) + assert all(valid(qi, reference_start + ki) for qi in (4, 5) for ki in (0, 1)) + assert not any(valid(qi, reference_start + ki) for qi in (0, 1) for ki in (0, 1)) + + +def test_in_context_attention_hands_flex_a_score_mod_only_when_log_scale_is_nonzero(): + # At log_scale 0 the bias is an exact identity, so flex must run without a score_mod; + # otherwise it must get the cached partial (compiled flex guards on its identity). + layout = _SHORT_CLIP_LAYOUT + captured = [] + + def fake_flex_attention(q, k, v, *, block_mask, score_mod): + captured.append(score_mod) + return torch.zeros_like(q) + + gen, reference = torch.zeros(1, 4, 2, 8), torch.zeros(1, 2, 2, 8) + with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention): + for log_scale in (0.0, -1.3): + WanAnimate2TransformerBlock._in_context_attention( + SimpleNamespace(log_scale=log_scale), + gen, + gen, + gen, + reference, + reference, + "SENTINEL_BLOCK_MASK", + layout, + ) + assert captured[0] is None + assert captured[1] is _make_score_mod(layout.max_tokens_per_frame, -1.3) + + +def test_make_score_mod_is_cached_by_key(): + # Compiled flex_attention guards on score_mod identity; a fresh closure per call + # would recompile every clip. + assert _make_score_mod(4, 0.0) is _make_score_mod(4, 0.0) + assert _make_score_mod(4, 0.0) is not _make_score_mod(4, 1.3) + assert _make_score_mod(4, 0.0) is not _make_score_mod(8, 0.0) + + +def test_score_mod_bias_region_matches_the_official_formula(): + # The official wan_animate_2_model.py adds log_scale to keys with index in [hw, 2*hw), + # hw = full-clip tokens per frame (external literal: the distilled checkpoint depends on + # it). In the padded [gen | reference-video] layout those are the keys of generation + # frame 1; the reference-video slice at padded_generation_video_len is not biased. + layout = _SHORT_CLIP_LAYOUT + hw = layout.max_tokens_per_frame # 4 + score_mod = _make_score_mod(hw, 5.0) + + def biased(kv_idx): + out = score_mod(torch.tensor([1.0]), 0, 0, 0, torch.tensor(kv_idx)) + return out.item() == 6.0 + + probes = list(range(0, 3 * hw)) + [layout.padded_generation_video_len] + assert [kv for kv in probes if biased(kv)] == [4, 5, 6, 7] + + # Base weights ship log_scale 0.0: the score_mod must then be an exact identity. + identity = _make_score_mod(hw, 0.0) + score = torch.randn(5) + for kv_idx in (0, hw, 2 * hw, layout.padded_generation_video_len): + assert torch.equal( + identity(score.clone(), 0, 0, 0, torch.tensor(kv_idx)), score + ) + + +class _FakeAllToAll4D: + """Single-process all_to_all_4D for SP=2: split scatter_dim into (SP, .) and fold + that factor onto gather_dim (and the inverse).""" + + SP = 2 + + def __init__(self): + self.calls = [] + + def __call__(self, x, scatter_dim, gather_dim): + self.calls.append((scatter_dim, gather_dim)) + b, s, h, c = x.shape + sp = self.SP + if (scatter_dim, gather_dim) == (2, 1): + assert h % sp == 0 + return ( + x.reshape(b, s, sp, h // sp, c) + .permute(0, 2, 1, 3, 4) + .reshape(b, sp * s, h // sp, c) + ) + if (scatter_dim, gather_dim) == (1, 2): + assert s % sp == 0 + return ( + x.reshape(b, sp, s // sp, h, c) + .permute(0, 2, 1, 3, 4) + .reshape(b, s // sp, sp * h, c) + ) + raise AssertionError((scatter_dim, gather_dim)) + + +def _reference_pass_stub(captured: list) -> SimpleNamespace: + # Minimal block for forward_ref: identity norm/modulation, q/k/v straight from the stream, + # attn1 records its keyword arguments. + def attn1(q, k, v, **kwargs): + captured.append({"q": q, "k": k, **kwargs}) + return q + + return SimpleNamespace( + local_num_heads=2, + _modulate=lambda t: tuple(torch.zeros(1, 1, 8) for _ in range(6)), + norm1=lambda x, shift, scale: x, + _qkv=lambda x, n: ( + x.reshape(1, -1, n, 4), + x.reshape(1, -1, n, 4) + 1, + x.reshape(1, -1, n, 4) + 2, + ), + attn1=attn1, + _cross_and_ffn=lambda hidden, attn, *rest: hidden, + ) + + +def _run_reference_pass( + stub, + *, + seq_len: int, + num_valid: int, + sp_size: int, + key_mask=None, + k_cache=None, + v_cache=None, + cos=None, + sin=None, +): + cos = torch.ones(num_valid, 2) if cos is None else cos + sin = torch.zeros(num_valid, 2) if sin is None else sin + with ( + patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size), + patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)), + patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x), + ): + return WanAnimate2TransformerBlock.forward_ref( + stub, + hidden_states=torch.arange(seq_len * 8, dtype=torch.float32).reshape( + 1, seq_len, 8 + ), + encoder_hidden_states=torch.zeros(1, 4, 8), + timestep_modulation=torch.zeros(1, 6, 8), + reference_video_rope_cos=cos, + reference_video_rope_sin=sin, + reference_video_num_tokens=num_valid, + index=0, + k_cache={} if k_cache is None else k_cache, + v_cache={} if v_cache is None else v_cache, + reference_video_key_mask=key_mask, + ) + + +@pytest.mark.parametrize("seq_len,sp_size", [(6, 1), (8, 2)]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_reference_cache_reuses_rotated_key_and_preserves_padding( + seq_len, sp_size, dtype +): + captured, keys, values = [], {}, {} + angles = torch.arange(12, dtype=torch.float32).view(6, 2) / 7 + stub = _reference_pass_stub(captured) + qkv = stub._qkv + raw_keys = [] + + def project(x, n): + projected = tuple(t.to(dtype) for t in qkv(x, n)) + raw_keys.append(projected[1]) + return projected + + stub._qkv = project + _run_reference_pass( + stub, + seq_len=seq_len, + num_valid=6, + sp_size=sp_size, + k_cache=keys, + v_cache=values, + cos=angles.cos(), + sin=angles.sin(), + ) + raw = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 2, 4) + expected = torch.cat( + [ + _apply_rope_interleaved( + (raw[:, :6] + 1).to(dtype), angles.cos(), angles.sin() + ), + (raw[:, 6:] + 1).to(dtype), + ], + dim=1, + ) + torch.testing.assert_close(keys[0], expected, rtol=0, atol=0) + torch.testing.assert_close(values[0], (raw + 2).to(dtype), rtol=0, atol=0) + torch.testing.assert_close(keys[0], captured[0]["k"], rtol=0, atol=0) + assert keys[0].data_ptr() == raw_keys[0].data_ptr() + + +def test_reference_pass_passes_no_key_mask_when_the_sequence_has_no_padding(): + # A dense all-true key mask would route the reference stream through SDPA instead of the + # configured attention backend; with no padding the pass must hand attn1 attn_mask=None and + # RoPE the whole sequence. + captured: list = [] + _run_reference_pass( + _reference_pass_stub(captured), seq_len=6, num_valid=6, sp_size=1 + ) + assert len(captured) == 1 + assert captured[0]["attn_mask"] is None + assert captured[0]["q"].shape == (1, 6, 2, 4) + + +@pytest.mark.parametrize("seq_len,sp_size", [(4, 1), (6, 2)]) +def test_generation_reuses_reference_cache_without_rotating_it(seq_len, sp_size): + stub = _reference_pass_stub([]) + captured = [] + + def attend(q, k, v, ref_k, ref_v, *rest): + captured.append((q, k, ref_k, ref_v)) + return q + + stub._in_context_attention = attend + keys = {0: torch.randn(1, 2, 2, 4)} + values = {0: torch.randn_like(keys[0])} + saved_key = keys[0].clone() + angles = torch.arange(8, dtype=torch.float32).view(4, 2) / 7 + hidden = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 8) + with ( + patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size), + patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)), + patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x), + ): + for _ in range(2): + WanAnimate2TransformerBlock.forward_gen( + stub, + hidden_states=hidden, + encoder_hidden_states=torch.zeros(1, 4, 8), + timestep_modulation=torch.zeros(1, 6, 8), + gen_cos=angles.cos(), + gen_sin=angles.sin(), + block_mask=None, + layout=_SHORT_CLIP_LAYOUT, + index=0, + k_cache=keys, + v_cache=values, + ) + raw = hidden.view(1, seq_len, 2, 4) + for q, k, ref_k, ref_v in captured: + for actual, source in ((q, raw), (k, raw + 1)): + expected = torch.cat( + [ + _apply_rope_interleaved(source[:, :4], angles.cos(), angles.sin()), + source[:, 4:], + ], + dim=1, + ) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + assert ref_k is keys[0] and ref_v is values[0] + torch.testing.assert_close(keys[0], saved_key, rtol=0, atol=0) + + +def test_reference_pass_masks_the_sp_tail_padding_with_the_per_clip_mask(): + # Gathered under SP the sequence carries tail padding: the pad keys are masked, and a mask + # built once per clip by the caller is used as is. + captured: list = [] + _run_reference_pass( + _reference_pass_stub(captured), seq_len=8, num_valid=6, sp_size=2 + ) + mask = captured[0]["attn_mask"] + assert mask is not None and mask.dtype == torch.bool and tuple(mask.shape) == (1, 8) + assert mask[0, :6].all() and not mask[0, 6:].any() + assert captured[0]["skip_sequence_parallel_override"] is True + + per_clip = torch.tensor([[True] * 6 + [False] * 2]) + captured.clear() + _run_reference_pass( + _reference_pass_stub(captured), + seq_len=8, + num_valid=6, + sp_size=2, + key_mask=per_clip, + ) + assert captured[0]["attn_mask"] is per_clip + + +def test_gather_then_scatter_round_trips(): + # _sp_gather_qkv: [B,S_local,N_tp,C] -> [B,S_full,N_local,C]; _sp_scatter_out inverts it. + fake = _FakeAllToAll4D() + with ( + patch( + f"{_BLOCK}._sp_gather_qkv", + side_effect=lambda q, k, v: fake( + torch.cat([q, k, v]), scatter_dim=2, gather_dim=1 + ).chunk(3), + ) as gather, + patch( + f"{_BLOCK}._usp_output_all_to_all", + side_effect=lambda x, head_dim: fake(x, scatter_dim=1, gather_dim=2), + ) as scatter, + ): + q = torch.arange(1 * 4 * 8 * 16, dtype=torch.float32).reshape(1, 4, 8, 16) + k = q + 1000.0 + v = q + 2000.0 + + gq, gk, gv = wan_animate_2_block._sp_gather_qkv(q, k, v) + assert tuple(gq.shape) == (1, 8, 4, 16) + assert tuple(gk.shape) == (1, 8, 4, 16) + assert tuple(gv.shape) == (1, 8, 4, 16) + + back = _sp_scatter_out(gq) + assert tuple(back.shape) == tuple(q.shape) + assert torch.equal(back, q) + gather.assert_called_once_with(q, k, v) + scatter.assert_called_once_with(gq, head_dim=2) + + assert fake.calls == [(2, 1), (1, 2)] + + +@pytest.mark.parametrize( + "full_clip_grid, expected", + [ + # 4 frames x 4 tokens: 16 gen + 12 reference-video tokens, one 128 block each. + ((4, 2, 2), (4, 16, 12, 128, 128, 256)), + # 4 frames x 100 tokens: 400 -> 512, 300 -> 384. + ((4, 10, 10), (100, 400, 300, 512, 384, 896)), + ], +) +def test_layout_pads_the_full_clip_geometry_to_128_token_blocks( + full_clip_grid, expected +): + # The reference video has no counterpart for the generation stream's reference-image + # frame, so its token count is one frame short of the generation stream's. + layout = _InContextLayout((3, 2, 2), (2, 2, 2), full_clip_grid) + assert ( + layout.max_tokens_per_frame, + layout.max_generation_video_num_tokens, + layout.max_reference_video_num_tokens, + layout.padded_generation_video_len, + layout.padded_reference_video_len, + layout.padded_kv_len, + ) == expected + + +class _KvTaggingTransformer: + """Fake DiT: ``build_reference_kv`` tags the K/V with the clip's reference-video latents, + ``__call__`` records the (clip_cond, reference_kv) pair each forward was handed.""" + + def __init__(self): + self.context_calls = 0 + self.forward_calls: list[ + tuple[WanAnimate2ClipConditioning, WanAnimate2ReferenceKV] + ] = [] + + def build_reference_kv(self, *, clip_cond): + latents = clip_cond.reference_video_latents + return WanAnimate2ReferenceKV(k={0: latents}, v={0: latents}) + + def prepare_context(self, prompt_embeddings, image_embeddings): + self.context_calls += 1 + return prompt_embeddings + + def __call__(self, *, hidden_states, clip_cond, reference_kv, **_unused): + self.forward_calls.append((clip_cond, reference_kv)) + return hidden_states.unsqueeze(0) + + +class _TwoStepScheduler: + timesteps = torch.tensor([1000.0, 500.0]) + order = 1 + + def set_timesteps(self, *, sigmas, device): + pass + + def step(self, noise_pred, t, sample, return_dict): + return (sample,) + + +def _clip_conditioning(reference_video_latents): + return WanAnimate2ClipConditioning( + reference_video_latents=reference_video_latents, + generation_video_grid_sizes=(2, 1, 1), + reference_video_frame_0_image_embeddings=torch.zeros(1, 1, 4), + reference_video_condition=torch.zeros(20, 1, 2, 2), + prompt_ref_embeddings=torch.zeros(1, 4), + generation_condition=torch.zeros(20, 2, 2, 2), + reference_video_grid_sizes=(1, 1, 1), + full_clip_grid_sizes=(2, 1, 1), + num_frames=4, + init_noise=torch.zeros(16, 2, 2, 2), + ) + + +def _batch_without_perf_recording(*, is_warmup=False) -> SimpleNamespace: + """The clip loop reads only the perf-recording fields off the batch (and hands it to + set_forward_context); no metrics object means no step records.""" + return SimpleNamespace( + metrics=None, + perf_dump_path=None, + is_warmup=is_warmup, + rollout=False, + return_trajectory_latents=False, + ) + + +def _fake_denoising_stage( + transformer, *, events: list[str] +) -> WanAnimate2DenoisingStage: + """Stand-in for WanAnimate2DenoisingStage in _denoise_single_clip; the residency hooks + record into ``events`` so a test can read the use interval back.""" + stage = WanAnimate2DenoisingStage.__new__(WanAnimate2DenoisingStage) + stage.transformer = transformer + stage.scheduler = _TwoStepScheduler() + stage.step_profile = lambda: None + stage._apply_nvtx_gate = lambda is_warmup: False + stage.log_info = lambda *args: None + stage.begin_declared_component_use = lambda *, component_name, module: ( + events.append(f"begin:{component_name}") + ) + stage._finish_active_component_use = lambda: events.append("finish") + return stage + + +def _single_clip_request_state(guidance_scale: float) -> SimpleNamespace: + return SimpleNamespace( + prompt_embeddings=torch.zeros(1, 4), + negative_prompt_embeddings=torch.zeros(1, 4), + reference_image_embeddings=torch.zeros(1, 1, 4), + inputs=SimpleNamespace(num_inference_steps=2, guidance_scale=guidance_scale), + ) + + +@pytest.mark.parametrize("is_warmup", [False, True]) +@pytest.mark.parametrize("guidance_scale", [1.0, 3.0]) +def test_each_clip_is_denoised_with_kv_from_its_own_reference_video( + is_warmup, guidance_scale +): + """Consecutive clips that share seed and clip index but not the reference video (two + requests on one DiT) must each be denoised with K/V built from their own reference video, + never with K/V left over from the previous one.""" + transformer = _KvTaggingTransformer() + events = [] + stage = _fake_denoising_stage(transformer, events=events) + stage.step_profile = lambda: events.append("profile") + server_args = SimpleNamespace( + pipeline_config=SimpleNamespace(flow_shift=5.0), + enable_cfg_parallel=False, + disable_autocast=False, + ) + request_state = _single_clip_request_state(guidance_scale=guidance_scale) + first = _clip_conditioning(torch.full((16, 1, 2, 2), 1.0)) + second = _clip_conditioning(torch.full((16, 1, 2, 2), 2.0)) + + with patch( + f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu") + ): + for clip_cond in (first, second): + WanAnimate2DenoisingStage._denoise_single_clip( + stage, + batch=_batch_without_perf_recording(is_warmup=is_warmup), + server_args=server_args, + request_state=request_state, + clip_condition=clip_cond, + ) + + # 2 clips x 2 steps; every forward of a clip gets the K/V built from that clip's latents. + branches = 2 if guidance_scale > 1 else 1 + assert len(transformer.forward_calls) == 4 * branches + assert transformer.context_calls == 2 * branches + assert events.count("profile") == (0 if is_warmup else 4) + for clip_cond, reference_kv in transformer.forward_calls: + assert reference_kv.k[0] is clip_cond.reference_video_latents + assert ( + transformer.forward_calls[0][1] + is not transformer.forward_calls[2 * branches][1] + ) + + +def test_dit_work_of_a_clip_runs_inside_one_declared_transformer_use(): + """The stage declares three component uses, so the residency manager does not activate + the DiT at stage entry; the reference pass and every CFG forward of a clip must run after + the transformer use begins and before it is finished, or an offloaded DiT is never + brought back to the device.""" + events: list[str] = [] + + class _EventTransformer(_KvTaggingTransformer): + def build_reference_kv(self, *, clip_cond): + events.append("reference_kv") + return super().build_reference_kv(clip_cond=clip_cond) + + def __call__(self, **kwargs): + events.append("forward") + return super().__call__(**kwargs) + + def prepare_context(self, *args): + events.append("context") + return super().prepare_context(*args) + + stage = _fake_denoising_stage(_EventTransformer(), events=events) + server_args = SimpleNamespace( + pipeline_config=SimpleNamespace(flow_shift=5.0), + enable_cfg_parallel=False, + disable_autocast=False, + ) + with patch( + f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu") + ): + WanAnimate2DenoisingStage._denoise_single_clip( + stage, + batch=_batch_without_perf_recording(), + server_args=server_args, + # guidance_scale 3.0: two forwards (cond, uncond) per step. + request_state=_single_clip_request_state(guidance_scale=3.0), + clip_condition=_clip_conditioning(torch.zeros(16, 1, 2, 2)), + ) + + assert events == [ + "begin:transformer", + "reference_kv", + "context", + "forward", + "context", + "forward", + "forward", + "forward", + "finish", + ] + + +def _ensure_single_process_parallel_runtime() -> None: + # The DiT's parallel linear layers need the TP group even on a meta device. + if model_parallel_is_initialized(): + return + ensure_distributed_env_defaults() + maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1) + + +def test_dit_constructs_only_its_own_blocks(): + """Every block the DiT constructs is the in-context block; the parent's block set is not + built and then replaced (40 block objects per construction, allocated on whatever device + the loader initializes on).""" + _ensure_single_process_parallel_runtime() + config = WanAnimate2Config() + config.arch_config.num_layers = 2 + constructed: list[type] = [] + original_init = WanTransformerBlock.__init__ + + def recording_init(self, *args, **kwargs): + constructed.append(type(self)) + original_init(self, *args, **kwargs) + + with ( + patch.object(WanTransformerBlock, "__init__", recording_init), + torch.device("meta"), + ): + model = WanAnimate2Transformer3DModel(config, hf_config={}) + + assert [type(block) for block in model.blocks] == [WanAnimate2TransformerBlock] * 2 + assert constructed == [WanAnimate2TransformerBlock] * 2 + + +def test_dit_inherits_shared_wan_projections_and_output_head(): + assert WanAnimate2TransformerBlock._qkv is WanTransformerBlock._qkv + assert ( + WanAnimate2TransformerBlock._cross_and_ffn is WanTransformerBlock._cross_and_ffn + ) + assert ( + WanAnimate2Transformer3DModel._unpatchify is WanTransformer3DModel._unpatchify + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py b/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py index 7e20507065..2f70a0a364 100644 --- a/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py +++ b/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py @@ -204,6 +204,39 @@ class TestFitAutoResidencyProbe: assert fitted.width * fitted.height <= 832 * 480 +class TestProbeFitTerminates: + def test_pinned_frame_count_ends_the_shrink_ladder(self): + """A model whose sampling params pin num_frames must not hang the probe fit. + + The lightened request comes back at the same size, so the fit must stop + instead of retrying the same shape forever. The request is larger than the + bounded warmup, so the measured-shape floor does not end the ladder first. + """ + from dataclasses import dataclass + + from sglang.multimodal_gen.runtime.managers.gpu_worker import ( + fit_auto_residency_probe, + ) + + @dataclass + class _PinnedFrames(SamplingParams): + def __post_init__(self) -> None: + self.num_frames = 37 + super().__post_init__() + + req = Req(sampling_params=_PinnedFrames(width=576, height=1024)) + fitted, estimate, steps = fit_auto_residency_probe( + req, + records=[_record(640, 800, 37, peak_gib=54.0)], + free_bytes=20 << 30, + total_bytes=80 << 30, + server_args=_server_args(), + ) + assert steps == 0 + assert (fitted.width, fitted.height, fitted.num_frames) == (576, 1024, 37) + assert estimate is not None and estimate > 20 << 30 + + class TestOutOfMemoryClassification: def test_allocation_failures_from_libraries_count_as_out_of_memory(self): from sglang.multimodal_gen.runtime.server_warmup import _is_out_of_memory