diff --git a/docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx b/docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx
new file mode 100644
index 0000000000..54f6c02582
--- /dev/null
+++ b/docs/cookbook/diffusion/Wan/Wan-Animate-2.mdx
@@ -0,0 +1,377 @@
+---
+title: Wan-Animate-2
+description: Run Wan-Animate-2 character animation (reference image + reference video) with SGLang Diffusion.
+metatags:
+ description: "Deploy and use the Wan2.2-Animate-2 14B character-animation model with SGLang Diffusion: reference-image + reference-video conditioning, single-expert native pipeline, config-file and /v1/videos invocation, and multi-GPU presets."
+---
+
+import { DiffusionModelTags } from '/src/snippets/diffusion/model-tags.jsx';
+
+
+
+## 1. Model Introduction
+
+[Wan-Animate-2](https://huggingface.co/Wan-AI/Wan2.2-Animate-2-14B-Diffusers) is a
+14B character-animation model in the Wan2.2 family. Instead of the text- or
+image-only conditioning of the other Wan2.2 checkpoints, it animates a **reference
+image** so that it follows the motion of a **reference video**, producing a new clip
+of that subject performing the reference video's motion.
+
+Use **`Wan-AI/Wan2.2-Animate-2-14B-Diffusers`** as `--model-path`. The distilled
+checkpoint (`Wan-AI/Wan2.2-Animate-2-14B-Distilled-Diffusers`, `log_scale=-1.3`,
+10 steps, no CFG) is not supported in this release.
+
+Unlike the A14B T2V/I2V checkpoints, Wan-Animate-2 is a **single-expert** model:
+it has one `guidance_scale` (default `3.0`), no low-noise expert
+(`guidance_scale_2`) and no boundary switching. Diffusers >= 0.40.0 serves
+Wan-Animate-2 as a modular pipeline (`WanAnimate2ModularPipeline`); SGLang
+implements the model natively and loads the same
+`Wan-AI/Wan2.2-Animate-2-14B-Diffusers` checkpoint. There is no diffusers backend
+fallback.
+
+
+Wan-Animate-2 is served through the native `WanAnimate2Pipeline`, selected
+automatically from the checkpoint's `model_index.json`. It takes **two** inputs, a
+reference image and a reference video, so it is invoked differently from the
+other Wan2.2 models. See [section 4](#4-model-invocation).
+
+
+### 1.1 Inputs
+
+| Input | Field | Passed via |
+| --- | --- | --- |
+| Reference image (the subject to animate) | `image_path` / `input_reference` | `--config` YAML for `sglang generate`; the `input_reference` part of a `POST /v1/videos` request |
+| Reference video (the motion to follow) | `video_path` | `--video-path` or `--config` YAML for `sglang generate`; the `video_path` field in the `/v1/videos` request body or `extra_body` |
+
+Both inputs are the base `image_path` / `video_path` sampling fields shared with the
+other video models; Wan-Animate-2 adds no model-specific input field. Offline runs
+pass both through a `--config` file (or `--image-path` / `--video-path`), and the
+online API forwards `video_path` into the pipeline while the reference image rides
+`input_reference`. Local paths must be readable by the server process. HTTP(S)
+video URLs use SGLang's shared media loader before decoding, without a disk cache.
+
+### 1.2 Defaults
+
+| Parameter | Default | Notes |
+| --- | --- | --- |
+| Resolution (`width`×`height`, the API's `size`) | `640`×`800` | A pixel-area budget, not the output dimensions: the frame keeps the reference image's aspect ratio and is the largest 16-aligned canvas with at most `width × height` pixels (see **Output resolution** below). `720`×`1280` is the larger supported budget. |
+| `clip_len` | `37` | Per-clip denoising chunk length; must be `4k+1` (VAE temporal stride 4), e.g. `37`, `65`, `81`. Other values are rounded to `4k+1` with the same `k` and the rounding is logged (`40` becomes `41`; `38` and `39` become `37`); values below `5` are rejected. |
+| `fps` | `16` | The reference video is resampled to this rate and the output MP4 is written at it; duration is preserved. |
+| `num_inference_steps` | `40` | Fewer steps (e.g. `20`) run faster. |
+| `guidance_scale` | `3.0` | Single expert; there is no `guidance_scale_2`. |
+| `prompt_ref` | `人物动作的参考视频` ("reference video of the character's motion") | Text for the reference-video branch. The official demo conditions that branch on this fixed description of the reference video rather than on the scene prompt; leave it unchanged unless reproducing a different reference setup. |
+| `enable_audio` | `true` | Keep the reference video's audio track on the output (see **Audio** below). Per request, like `clip_len`. |
+
+**Output length.** The output has as many frames as the (resampled) reference
+video, not `clip_len`. The video is generated clip by clip with `clip_len` frames
+per chunk; a larger `clip_len` improves temporal coherence at the cost of VRAM.
+`clip_len=81` roughly doubles the attention tokens of the default and OOMs a
+single 80 GB GPU. `num_frames` (and the API's `seconds`, which maps to it) does
+not control the length: it is ignored and the server logs that it was, so leave
+it unset.
+
+**Output resolution.** `width`×`height` (the API's `size`) is a pixel-area budget,
+not the output dimensions. The reference image's aspect ratio is kept, the frame is
+the largest canvas with at most `width × height` pixels whose sides are multiples of
+16, and the reference image and the driving-video frames are letterboxed onto it;
+the output MP4 has the canvas size. At the default `640`×`800` (512,000 pixels) a
+9:16 reference gives `528`×`944`, a 1:1 reference gives `704`×`704`, and only a 4:5
+reference gives exactly `640`×`800`. To obtain a specific output size, request a
+budget with the same aspect ratio as the reference image (for a 9:16 image,
+`576`×`1024` maps onto itself). This differs from the OpenAI Videos API and from the
+other SGLang diffusion pipelines, where `size` is the literal output resolution.
+
+**Audio.** The output MP4 carries the reference video's audio track, as the
+official pipeline does. Set `enable_audio=false` on a request (a `--config` field
+offline, a form field or `extra_body` entry online) for a silent output. If the
+track cannot be decoded, the output is silent and the server logs one warning per
+process; a reference video without an audio track also gives a silent output.
+
+**Lossless conditioning reuse.** Reference-video keys are rotated once and cached per clip, then reused by all
+denoising steps and both CFG branches. This removes repeated RoPE computation
+without changing the attention mask, precision, or sampling schedule. The cache
+uses the existing key storage and is released after the clip; it is not an
+approximate timestep-skipping cache.
+Clips containing only padding that would be removed from the final video are not
+denoised. The padding and conditioning of every retained clip are unchanged.
+
+Text encoding uses the native `TextEncodingStage` and its conditioning cache and
+residency management. CFG-parallel dispatch shares the native CFG utilities while
+retaining the single-GPU arithmetic order.
+Each clip uses the shared denoising loop for step profiling, NVTX ranges, progress,
+and DiT release. Reference K/V preparation and clip-specific prediction remain
+model hooks; clip decoding still feeds the next clip's conditioning. This does
+not enable cache-dit or breakable CUDA graphs for this model.
+Clip decoding uses the shared `DecodingStage.decode_raw` path for VAE compilation,
+tiling and OOM diagnostics. Raw decoded pixels are kept for the next clip: no
+extra conversion to and from `[0, 1]` is introduced. The adapter preserves the
+existing FP32 VAE execution and per-channel latent scaling.
+QKV projections, cross-attention, FFN and the output head are inherited from the
+native Wan implementation. Ulysses exchanges use the shared packed-QKV path;
+the model-specific reference attention mask and K/V lifecycle are retained.
+Text and image conditioning projections are reused within each clip and CFG
+branch, rather than recomputed at every timestep. These projected tensors are
+released after the clip and never reused across requests. Decoded output is
+copied to CPU and clip-local intermediates are released; only the decoded
+overlap frames needed by the next clip remain on the GPU.
+Geometry caches retain at most four RoPE tables and two attention masks, rather
+than accumulating GPU tensors for every request size. Evicted entries are
+recomputed without changing the results; frequently alternating among more
+geometries can add setup work.
+
+Measured on one H200, with a 576×1024 budget, 8 steps, CFG 3, seed 42,
+`clip_len=37`, text-encoder CPU offload, and MP4 plus raw-frame output, the initial
+reuse changes reduced warm request latency from 42.30s to 41.34s (two requests
+per variant). With `clip_len=17`, omitting a fully cropped trailing clip reduced
+59.60s to 40.45s (one warm request per variant). Raw RGB frames were bit-identical
+and peak allocated VRAM did not increase. These are workload-specific results,
+not general speedup claims; both comparisons used PyTorch 2.13.0+cu130,
+Transformers 5.12.1, and Diffusers 0.37.0. The baseline was the initial native
+implementation, not the official Diffusers implementation.
+
+## 2. SGLang-diffusion Installation
+
+SGLang-diffusion offers multiple installation methods. You can choose the most
+suitable installation method based on your hardware platform and requirements.
+
+Please refer to the [official SGLang-diffusion installation guide](/docs/sglang-diffusion/installation)
+for installation instructions.
+
+## 3. Model Deployment
+
+### 3.1 Basic Configuration
+
+On a single high-VRAM GPU, launch the native pipeline with:
+
+```bash Command
+sglang serve \
+ --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \
+ --port 30010
+```
+
+Routing to the native Wan-Animate-2 pipeline is automatic from the checkpoint's
+`model_index.json` (`_class_name: WanAnimate2Pipeline`); pass
+`--pipeline-class-name WanAnimate2Pipeline` only for a local mirror that lacks
+`model_index.json`.
+
+### 3.2 Configuration Tips
+
+See [Performance Optimization](/docs/sglang-diffusion/performance-optimization) for
+acceleration features and their runtime requirements.
+
+- `--num-gpus {NUM_GPUS}`: Number of GPUs to use.
+- `--tp-size {TP_SIZE}`: Tensor parallelism size.
+- `--ulysses-degree {ULYSSES_DEGREE}`: DeepSpeed-Ulysses-style sequence parallelism (USP). The **ring degree must be 1** for this model; `--ring-degree` greater than 1 is rejected with `NotImplementedError`. Combines with `--tp-size`: Ulysses splits the TP-local attention heads, so `--tp-size` must divide the 40 heads and `--ulysses-degree` must divide the per-shard count (see [section 3.3](#3-3-multi-gpu-and-memory-presets)).
+- `--enable-cfg-parallel`: Split the guided and unguided branches across GPUs.
+- `--text-encoder-cpu-offload`: CPU-offload the text encoder to save memory; it is moved to the GPU for the one encode per request.
+
+### 3.3 Multi-GPU and memory presets
+
+| Target | Recommended server flags | Notes |
+| --- | --- | --- |
+| 1 GPU, 96 GB or more | *(no extra flags)* | DiT resident: about 70 GiB peak per GPU at the default workload. Add `--text-encoder-cpu-offload` to keep the fp32 text encoder off the GPU. |
+| 1 GPU, 80 GB | `--dit-layerwise-offload --layerwise-offload-components transformer` | Streams the DiT layer by layer: about 40 GiB peak at the same end-to-end time as the resident DiT; output identical. |
+| 2+ GPUs, long sequences | `--num-gpus N --ulysses-degree N` | Sequence parallel across ranks (ring degree stays 1). |
+| 2+ GPUs, large DiT | `--num-gpus N --tp-size N` | Tensor parallel across attention heads. With `--tp-size` smaller than `--num-gpus` (or than half of it with `--enable-cfg-parallel`), the leftover GPUs are auto-assigned to Ulysses sequence parallelism (last row). |
+| 2 GPUs, split CFG | `--num-gpus 2 --enable-cfg-parallel` | Splits the guided/unguided branches. |
+| 2 GPUs, 80 GB cards | `--num-gpus 2 --enable-cfg-parallel --dit-layerwise-offload --layerwise-offload-components transformer --encoder-parallel replicate --vae-config.use-parallel-decode false` with `SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D=1` | Same speed as resident CFG-parallel at about 43 GiB per GPU, output bit-identical to a single GPU (the CI configuration). |
+| 4 GPUs | `--num-gpus 4 --enable-cfg-parallel --ulysses-degree 2` | CFG-parallel across two pairs of Ulysses ranks: the fastest 4-GPU layout; pass the flags explicitly to pin the layout instead of relying on the automatic assignment. Ulysses shards the attention sequence, so 4-GPU outputs match the single-GPU video to about 30 dB mean PSNR rather than bit for bit. On 80 GB cards add `--dit-layerwise-offload --layerwise-offload-components transformer` (about 35 GiB per GPU, same speed). |
+| FSDP sharding | `--use-fsdp-inference` is **not supported** | Rejected at startup, before any weights load; use `--tp-size` to shard the DiT or `--dit-layerwise-offload` to cut peak memory. |
+| 4+ GPUs, TP x Ulysses | `--num-gpus 4 --tp-size 2 --ulysses-degree 2` | Tensor parallel across attention heads, Ulysses across the TP-local heads: `--tp-size` must divide the 40 heads and `--ulysses-degree` the per-shard count (`40 / tp`). The memory-leaning 4-GPU layout: about 33 GiB peak per GPU (25 GiB torch reserved), the lowest 4-GPU peak after `--tp-size 4`, at about 13 percent slower than CFG-parallel x tensor parallel and 27 percent slower than the default CFG-parallel x Ulysses layout; use the default for speed and this layout for memory. CFG-parallel can be added on top with twice the GPUs (`--num-gpus 8 --enable-cfg-parallel --tp-size 2 --ulysses-degree 2`). |
+
+## 4. Model Invocation
+
+### 4.1 Generate offline with a config file
+
+The offline path passes both inputs (and the sampling parameters) through a
+`--config` YAML:
+
+```yaml wan_animate_2_run.yaml
+prompt: "a person dancing"
+image_path: /path/to/reference.png
+video_path: /path/to/reference_video.mp4
+clip_len: 37
+height: 800
+width: 640
+num_inference_steps: 40
+guidance_scale: 3.0
+fps: 16
+seed: 42
+```
+
+```bash Command
+sglang generate \
+ --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \
+ --config wan_animate_2_run.yaml \
+ --num-gpus 1 --text-encoder-cpu-offload --pin-cpu-memory \
+ --save-output --output-path outputs --output-file-name wan_animate_2.mp4
+```
+
+Any value in the YAML can also be passed as a CLI flag (for example
+`--image-path`, `--video-path`, `--num-inference-steps`, `--guidance-scale`,
+`--seed`).
+
+### 4.2 Serve and request over HTTP
+
+`POST /v1/videos` creates an asynchronous job. Send the **reference image** as the
+`input_reference` part and the **reference video** as `video_path`, then poll
+the job and download the finished MP4. `size` is the pixel-area budget described in
+[section 1.2](#1-2-defaults); the output dimensions follow the reference image's
+aspect ratio.
+
+```bash Command
+job_id=$(curl -sS -X POST http://127.0.0.1:30010/v1/videos \
+ --form-string "prompt=a person dancing" \
+ --form-string "video_path=/path/to/reference_video.mp4" \
+ --form-string "clip_len=37" \
+ --form-string "size=640x800" \
+ --form-string "num_inference_steps=40" \
+ --form-string "guidance_scale=3.0" \
+ --form-string "fps=16" \
+ --form-string "seed=42" \
+ --form "input_reference=@/path/to/reference.png;type=image/png" \
+ | python -c 'import json, sys; print(json.load(sys.stdin)["id"])')
+
+while true; do
+ status=$(curl -sS "http://127.0.0.1:30010/v1/videos/${job_id}" \
+ | python -c 'import json, sys; print(json.load(sys.stdin)["status"])')
+ [ "$status" = "completed" ] && break
+ [ "$status" = "failed" ] && exit 1
+ sleep 1
+done
+
+curl -sS -L "http://127.0.0.1:30010/v1/videos/${job_id}/content" \
+ -o wan_animate_2.mp4
+```
+
+The same request from the OpenAI Python client. The reference image is the
+`input_reference` file, and `video_path` (a path the server can read) and
+`clip_len` ride `extra_body`:
+
+```python Python
+from openai import OpenAI
+
+client = OpenAI(api_key="EMPTY", base_url="http://127.0.0.1:30010/v1")
+video = client.videos.create(
+ model="Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
+ prompt="a person dancing",
+ input_reference=open("/path/to/reference.png", "rb"),
+ extra_body={
+ "video_path": "/path/to/reference_video.mp4",
+ "clip_len": 37,
+ "guidance_scale": 3.0,
+ "enable_audio": False, # silent output; the reference video's audio is kept by default
+ },
+)
+```
+
+For more API usage and request examples, see the
+[SGLang Diffusion OpenAI API](/docs/sglang-diffusion/api/openai_api) reference.
+
+### 4.3 Reducing memory
+
+When a single GPU is tight on VRAM, stream the DiT one layer at a time instead of
+resident-loading it:
+
+```bash Command
+sglang serve \
+ --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \
+ --dit-layerwise-offload \
+ --layerwise-offload-components transformer \
+ --port 30010
+```
+
+Combine with `--text-encoder-cpu-offload --pin-cpu-memory` to also keep the text
+encoder off the GPU between uses. Lowering `clip_len` also reduces peak memory.
+
+## 5. Benchmark
+
+Test Environment:
+
+- Hardware: NVIDIA GB200 GPU (1x, 2x and 4x; 189 GB per GPU)
+- Model: Wan-AI/Wan2.2-Animate-2-14B-Diffusers
+- sglang diffusion version: 0.5.22
+- Workload: `size` 640x800 (pixel-area budget) with a 16:9 reference image, so the frame is 960x528; a 237-frame reference video at `fps` 30 (7 clips of `clip_len` 37); 40 steps, CFG 3.0, seed 123; `--text-encoder-cpu-offload --warmup-mode request`. E2E is the request time after the warmup request, so it excludes the one-off flex-attention compile.
+
+`bench_serving` cannot pass `video_path`, so the benchmark runs offline
+with `sglang generate --config` (the YAML from [section 4.1](#4-1-generate-offline-with-a-config-file))
+and reads timings from `--perf-dump-path`. `SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1`
+synchronizes the device so the per-stage timings are accurate.
+
+
+
+ **Benchmark Command** (1 GPU shown; add the flags from the table for the other rows):
+ ```shell Command
+ SGLANG_DIFFUSION_SYNC_STAGE_PROFILING=1 sglang generate \
+ --model-path Wan-AI/Wan2.2-Animate-2-14B-Diffusers \
+ --config wan_animate_2_run.yaml \
+ --num-gpus 1 --text-encoder-cpu-offload --warmup-mode request \
+ --save-output --perf-dump-path wan_animate_2_perf.json
+ ```
+
+ **Result** (peak memory per GPU from `nvidia-smi`):
+
+ | Config | GPUs | Extra flags | E2E (s) | Peak per GPU (GiB) |
+ | --- | ---: | --- | ---: | ---: |
+ | 1 GPU, DiT resident | 1 | none | 993 | 69.5 |
+ | 1 GPU, layerwise DiT offload | 1 | `--dit-layerwise-offload --layerwise-offload-components transformer` | 996 | 40.2 |
+ | 2 GPUs, CFG-parallel | 2 | `--enable-cfg-parallel` | 513 | 67.3 |
+ | 2 GPUs, CFG-parallel, replicated encoders, serial VAE decode | 2 | `--enable-cfg-parallel --encoder-parallel replicate --vae-config.use-parallel-decode false` | 515 | 71.0 |
+ | 2 GPUs, CFG-parallel, layerwise DiT offload, bit-identical to 1 GPU | 2 | `--enable-cfg-parallel --dit-layerwise-offload --layerwise-offload-components transformer --encoder-parallel replicate --vae-config.use-parallel-decode false` with `SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D=1` | 512 | 43.0 |
+ | 2 GPUs, Ulysses | 2 | `--ulysses-degree 2` | 547 | 58.1 |
+ | 2 GPUs, tensor parallel | 2 | `--tp-size 2` | 613 | 36.6 |
+ | 4 GPUs, framework default (CFG2 x SP2, KV-gather exchange) | 4 | none | 283 | 58.4 |
+ | 4 GPUs, CFG-parallel x Ulysses | 4 | `--enable-cfg-parallel --ulysses-degree 2` | 283 | 58.4 |
+ | 4 GPUs, CFG-parallel x Ulysses, layerwise DiT offload | 4 | `--enable-cfg-parallel --ulysses-degree 2 --dit-layerwise-offload --layerwise-offload-components transformer --encoder-parallel replicate --vae-config.use-parallel-decode false` | 291 | 35.2 |
+ | 4 GPUs, Ulysses | 4 | `--ulysses-degree 4` | 317 | 49.0 |
+ | 4 GPUs, CFG-parallel x tensor parallel | 4 | `--enable-cfg-parallel --tp-size 2` | 318 | 37.5 |
+ | 4 GPUs, tensor parallel x Ulysses | 4 | `--tp-size 2 --ulysses-degree 2` | 361 | 33.5 |
+ | 4 GPUs, tensor parallel | 4 | `--tp-size 4` | 379 | 27.3 |
+
+ Layerwise DiT offload cuts the 1-GPU peak by 29 GiB at the same end-to-end time.
+ The fastest 4-GPU layout is 3.5x faster than one GPU. The framework default and the explicit CFG-parallel x Ulysses
+ layout run at the same speed and produce the same output; the explicit flags pin the layout.
+ Tensor parallel x Ulysses is the memory-leaning 4-GPU layout: the lowest 4-GPU peak after `--tp-size 4`, about 13 percent slower than CFG-parallel x tensor parallel and 27 percent slower than the default.
+ Adding `--encoder-parallel replicate --vae-config.use-parallel-decode false` to the CFG-parallel x Ulysses row costs about 1 percent (287 s, 62.5 GiB) and does not make the output bit-identical to a single GPU: every Ulysses layout matches the single-GPU video to about 30 dB mean PSNR.
+ The layerwise rows are the configurations for 80 GB GPUs. With the replicated encoders, serial VAE decode and the
+ channels-last VAE layout (`SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D=1`) the 2-GPU CFG-parallel output is bit-identical to a single GPU.
+
+
+
+### 5.1 Native regression coverage
+
+A separate regression used two B300 GPUs, CFG-parallel with replicated encoders
+and serial VAE decode, a 576x1024 area budget, 40 steps, `clip_len=37`, CFG=3,
+and seed 42. The driving input was looped to 64 frames to exercise two clips;
+this is a bounded regression workload, not a long-video quality benchmark.
+Both revisions (`4ee69b4cf9ea` and `04725c1c2900`) used PyTorch 2.14.1+cu130,
+Transformers 5.19.0, Diffusers 0.37.0, and FlashAttention 4.0.0b34 on the same node.
+
+Cold and warm raw RGB outputs were byte-identical to the pre-optimization native
+implementation, both with resident DiT weights and with layerwise DiT offload.
+The single warm samples measured 112.29 s before and 112.40 s after optimization;
+offload measured 113.46 s. This does not establish a significant latency change.
+Rank 0's request-stage peak allocated memory was 52.24, 52.23, and 23.93 GiB,
+respectively; these are not loading peaks or whole-device physical memory peaks.
+
+An H200 HTTP smoke test also covered image upload, asynchronous job polling and
+download, request warmup, repeated requests, URL video input, and reference-audio
+passthrough. Repeated decoded RGB outputs were identical.
+
+These checks preserve the existing native output, not bitwise parity with the
+newer Diffusers modular pipeline. The native sampler follows the original Wan
+sigma grid starting at exactly 1.0 and retains its BF16 conditioning conversion;
+the modular pipeline's sampling and preprocessing differ. Official-reference
+accuracy must be validated separately with pinned versions and aligned settings.
+
+## 6. Troubleshooting
+
+- **OOM**: lower `clip_len` (for example `37` instead of `65`/`81`) or add `--dit-layerwise-offload --layerwise-offload-components transformer`.
+- **`--ring-degree` greater than 1** is rejected at startup with `NotImplementedError`. Use `--ulysses-degree` for sequence parallelism and keep the ring degree at 1.
+- **`--ulysses-degree` that does not divide the TP-local attention heads** (40 / `--tp-size`; with TP 1, degrees 3, 6, 16, ... are rejected) fails at startup with `ValueError`; with TP 1 use 2, 4, 5, 8, 10, 20 or 40. The same check covers leftover GPUs auto-assigned to sequence parallelism (for example `--num-gpus 8 --tp-size 2` runs TP 2 x Ulysses 4).
+- **FSDP** (`--use-fsdp-inference`) is not supported for this model and is rejected at startup; use `--tp-size` or DiT layerwise offload for memory.
+- **`--attention-backend`** (or `--component-attention-backends transformer=...`) is rejected at startup with `ValueError`: the in-context self-attention runs on torch `flex_attention` only. The text encoder still accepts `--component-attention-backends text_encoder=`.
+- **`enable_teacache` / `enable_spectrum`** request fields are rejected with `ValueError`: every block runs at every step, so the cache heuristics do not apply.
+- **`SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION`** is rejected at startup with `ValueError`: this model does not apply the offline Q/K rotation.
+- **Silent output**: the reference video's audio track is kept only when it can be decoded; otherwise the output is silent and the server logs one warning per process naming the cause. A reference video without an audio track also gives a silent output (one info line). Pass `enable_audio=false` to skip audio extraction altogether.
+- **Reference video not found**: `video_path` must be a path the server process can read. Pass it as `--video-path` or through `--config` offline, or in the request body online.
diff --git a/docs/docs.json b/docs/docs.json
index a99a0ab67d..decf40dade 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -1525,7 +1525,8 @@
"group": "Wan",
"pages": [
"cookbook/diffusion/Wan/Wan2.1",
- "cookbook/diffusion/Wan/Wan2.2"
+ "cookbook/diffusion/Wan/Wan2.2",
+ "cookbook/diffusion/Wan/Wan-Animate-2"
]
},
{
diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx
index 9d0d2e82b9..45d1bfc7f9 100644
--- a/docs/docs/sglang-diffusion/api/openai_api.mdx
+++ b/docs/docs/sglang-diffusion/api/openai_api.mdx
@@ -319,6 +319,21 @@ curl -sS -X POST "http://localhost:30010/v1/videos" \
}'
```
+
+`size` is the literal output resolution for most pipelines. Wan-Animate-2
+treats it as a pixel-area budget instead: the output keeps the reference image's
+aspect ratio and uses the largest frame with at most `width × height` pixels whose
+sides are multiples of 16, so `size=640x800` with a 9:16 reference image yields a
+528×944 video. Request a budget with the reference image's aspect ratio to get a
+specific size. Wan-Animate-2 also ignores `seconds` and `num_frames`: the output
+has as many frames as the (resampled) reference video, generated in chunks of
+`clip_len` frames; see the
+[Wan-Animate-2 cookbook](/cookbook/diffusion/Wan/Wan-Animate-2#1-2-defaults).
+Wan-Animate-2 keeps the reference video's audio track on the output; the
+model-specific `enable_audio` field (form field or `extra_body`, default `true`)
+set to `false` gives a silent output.
+
+
#### MP4 encoding
`x264_preset` picks the libx264 preset for the returned MP4: `ultrafast`,
diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx
index 2bea4493f3..c35d0ee06b 100644
--- a/docs/src/snippets/diffusion/model-catalog.jsx
+++ b/docs/src/snippets/diffusion/model-catalog.jsx
@@ -173,6 +173,11 @@ export const DiffusionModelCatalog = ({ category }) => {
],
cookbook: "/cookbook/diffusion/Wan/Wan2.2",
},
+ {
+ name: "Wan-Animate-2",
+ modelIds: ["Wan-AI/Wan2.2-Animate-2-14B-Diffusers"],
+ cookbook: "/cookbook/diffusion/Wan/Wan-Animate-2",
+ },
{
name: "FastWan / TurboWan",
modelIds: [
diff --git a/python/sglang/multimodal_gen/configs/models/dits/__init__.py b/python/sglang/multimodal_gen/configs/models/dits/__init__.py
index 7b4d0fe3fa..c87cc5ab8b 100644
--- a/python/sglang/multimodal_gen/configs/models/dits/__init__.py
+++ b/python/sglang/multimodal_gen/configs/models/dits/__init__.py
@@ -22,6 +22,7 @@ from sglang.multimodal_gen.configs.models.dits.sana_video import SanaVideoConfig
from sglang.multimodal_gen.configs.models.dits.stablediffusion3 import (
StableDiffusion3TransformerConfig,
)
+from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoConfig
__all__ = [
@@ -34,6 +35,7 @@ __all__ = [
"LingBotVideoMoEConfig",
"LongLive2VideoConfig",
"MiniMaxH3DiTConfig",
+ "WanAnimate2Config",
"WanVideoConfig",
"Hunyuan3DDiTConfig",
"MOVAAudioConfig",
diff --git a/python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py b/python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py
new file mode 100644
index 0000000000..c3d5ddcd54
--- /dev/null
+++ b/python/sglang/multimodal_gen/configs/models/dits/wan_animate_2.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Wan-Animate-2 DiT config. The checkpoint is the Wan2.2-I2V-14B backbone with no
+# extra parameter tensors, so WanVideoArchConfig is reused with in_channels=36,
+# image_dim=1280, added_kv_proj_dim=5120 and a param-name remap.
+from dataclasses import dataclass, field
+
+from sglang.multimodal_gen.configs.models.dits.base import DiTConfig
+from sglang.multimodal_gen.configs.models.dits.wanvideo import WanVideoArchConfig
+
+# Checkpoint block params use the official Wan names nested under `blocks.{i}.block.*`.
+# The loader applies mapping rules in insertion order, re-scanning after each hit, so the
+# extras below chain into the reused Wan lora (official -> hf) and main (hf -> native) maps.
+_WAN = WanVideoArchConfig()
+
+
+_WAN_ANIMATE_2_EXTRA_MAPPING: dict[str, str] = {
+ # Strip the `.block` infix so the reused Wan rules match.
+ r"^blocks\.(\d+)\.block\.(.*)$": r"blocks.\1.\2",
+ # Self-attn / cross-attn qk-norms (not covered by the Wan lora map).
+ r"^blocks\.(\d+)\.self_attn\.norm_q\.(.*)$": r"blocks.\1.norm_q.\2",
+ r"^blocks\.(\d+)\.self_attn\.norm_k\.(.*)$": r"blocks.\1.norm_k.\2",
+ r"^blocks\.(\d+)\.cross_attn\.norm_q\.(.*)$": r"blocks.\1.attn2.norm_q.\2",
+ r"^blocks\.(\d+)\.cross_attn\.norm_k\.(.*)$": r"blocks.\1.attn2.norm_k.\2",
+ # Cross-attn I2V image branch -> WanI2VCrossAttention add_k_proj/add_v_proj/norm_added_k.
+ r"^blocks\.(\d+)\.cross_attn\.k_img\.(.*)$": r"blocks.\1.attn2.add_k_proj.\2",
+ r"^blocks\.(\d+)\.cross_attn\.v_img\.(.*)$": r"blocks.\1.attn2.add_v_proj.\2",
+ r"^blocks\.(\d+)\.cross_attn\.norm_k_img\.(.*)$": r"blocks.\1.attn2.norm_added_k.\2",
+ # The Diffusers checkpoint keeps the official self_attn/cross_attn prefixes but names
+ # the projections the diffusers way (to_q/to_k/to_v/to_out.0, add_*_proj).
+ r"^blocks\.(\d+)\.self_attn\.to_out\.0\.(.*)$": r"blocks.\1.to_out.\2",
+ r"^blocks\.(\d+)\.self_attn\.(to_q|to_k|to_v)\.(.*)$": r"blocks.\1.\2.\3",
+ r"^blocks\.(\d+)\.cross_attn\.to_out\.0\.(.*)$": r"blocks.\1.attn2.to_out.\2",
+ r"^blocks\.(\d+)\.cross_attn\.(to_q|to_k|to_v|add_k_proj|add_v_proj|norm_added_k)\.(.*)$": r"blocks.\1.attn2.\2.\3",
+ # `norm3` is the block's only affine LayerNorm and feeds cross-attn; the reused rules
+ # only know its hf name (`norm2`).
+ r"^blocks\.(\d+)\.norm3\.(.*)$": r"blocks.\1.self_attn_residual_norm.norm.\2",
+ # Per-block 6-way modulation -> scale_shift_table.
+ r"^blocks\.(\d+)\.modulation$": r"blocks.\1.scale_shift_table",
+ # Output head (`head.norm` is param-free; no-op if absent).
+ r"^head\.head\.(.*)$": r"proj_out.\1",
+ r"^head\.modulation$": r"scale_shift_table",
+ r"^head\.norm\.(.*)$": r"norm_out.\1",
+ # Embedders in official naming; hf-named checkpoints are covered by the reused rules.
+ r"^text_embedding\.0\.(.*)$": r"condition_embedder.text_embedder.fc_in.\1",
+ r"^text_embedding\.2\.(.*)$": r"condition_embedder.text_embedder.fc_out.\1",
+ r"^time_embedding\.0\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_in.\1",
+ r"^time_embedding\.2\.(.*)$": r"condition_embedder.time_embedder.mlp.fc_out.\1",
+ r"^time_projection\.1\.(.*)$": r"condition_embedder.time_modulation.linear.\1",
+ r"^img_emb\.proj\.0\.(.*)$": r"condition_embedder.image_embedder.norm1.\1",
+ r"^img_emb\.proj\.1\.(.*)$": r"condition_embedder.image_embedder.ff.fc_in.\1",
+ r"^img_emb\.proj\.3\.(.*)$": r"condition_embedder.image_embedder.ff.fc_out.\1",
+ r"^img_emb\.proj\.4\.(.*)$": r"condition_embedder.image_embedder.norm2.\1",
+ # patch_embedding, self/cross-attn q/k/v/o and ffn.* are covered by the reused Wan maps.
+}
+
+
+@dataclass
+class WanAnimate2ArchConfig(WanVideoArchConfig):
+ """Wan2.2-I2V-14B backbone dims (inherited) with the Wan-Animate-2 I/O + remap."""
+
+ # Extras first, then Wan lora (official -> hf), then Wan main (hf -> native).
+ param_names_mapping: dict[str, str] = field(
+ default_factory=lambda: {
+ **_WAN_ANIMATE_2_EXTRA_MAPPING,
+ **_WAN.lora_param_names_mapping,
+ **_WAN.param_names_mapping,
+ }
+ )
+ # Only used for re-serialization, which the inference path does not exercise.
+ reverse_param_names_mapping: dict[str, str] = field(
+ default_factory=lambda: dict(_WAN.reverse_param_names_mapping)
+ )
+ # LoRA adapters use the official Wan names; the loader applies this before
+ # param_names_mapping.
+ lora_param_names_mapping: dict[str, str] = field(
+ default_factory=lambda: dict(_WAN.lora_param_names_mapping)
+ )
+
+ # in_channels: 16 noise + 20 conditioning (4-ch i2v mask + 16-ch cond latent).
+ # image_dim: CLIP (XLM-RoBERTa ViT-H/14) embed dim -> image_embedder.
+ # added_kv_proj_dim -> WanI2VCrossAttention (add_k_proj/add_v_proj/norm_added_k).
+ in_channels: int = 36
+ out_channels: int = 16
+ image_dim: int | None = 1280
+ added_kv_proj_dim: int | None = 5120
+ num_layers: int = 40
+
+ # In-context mechanism (no extra params): reference-video RoPE offsets and the flex-attention
+ # score_mod, consumed by forward_ref / forward_gen. The refer_* / log_scale names
+ # match the official and diffusers transformer configs; do not rename. In those
+ # names "refer" means the reference video (the motion source), not the reference image.
+ refer_offset_t: int = 1
+ refer_offset_h: int = 0
+ refer_offset_w: int = (
+ -1
+ ) # <0: resolved to the gen grid width in tokens at runtime (official convention)
+ refer_stride: int = 1
+ # Additive attention-score bias on the keys of generation frame 1 (official name);
+ # 0.0 disables it. Read from transformer/config.json when present; the base weights omit it.
+ log_scale: float = 0.0
+ # The image-embedding cross-attention (added_kv_proj_dim) is always built, as in the
+ # official model; the checkpoint config states the same and nothing else reads this.
+ use_img_emb: bool = True
+
+ def __post_init__(self) -> None:
+ # transformer/config.json uses the official Wan names for the backbone dims;
+ # update_model_arch parks unknown keys in extra_attrs, so translate them here.
+ extras = self.extra_attrs
+ if "num_heads" in extras:
+ self.num_attention_heads = extras.pop("num_heads")
+ if "dim" in extras:
+ self.attention_head_dim = extras.pop("dim") // self.num_attention_heads
+ if "in_dim" in extras:
+ self.in_channels = extras.pop("in_dim")
+ if "out_dim" in extras:
+ self.out_channels = extras.pop("out_dim")
+ if not self.use_img_emb:
+ raise ValueError(
+ "Wan-Animate-2 always builds the image-embedding cross-attention; "
+ "a checkpoint with use_img_emb=false is a different model."
+ )
+ self.patch_size = tuple(self.patch_size)
+ super().__post_init__()
+
+
+@dataclass
+class WanAnimate2Config(DiTConfig):
+ arch_config: WanAnimate2ArchConfig = field(default_factory=WanAnimate2ArchConfig)
+ prefix: str = "WanAnimate2"
diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py
index ddcd00feea..6ebf12c1cb 100644
--- a/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py
+++ b/python/sglang/multimodal_gen/configs/pipeline_configs/wan.py
@@ -3,11 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
from collections.abc import Callable
from dataclasses import dataclass, field
+from typing import Any
import torch
+from sglang.multimodal_gen import envs
from sglang.multimodal_gen.configs.models import DiTConfig, EncoderConfig, VAEConfig
-from sglang.multimodal_gen.configs.models.dits import WanVideoConfig
+from sglang.multimodal_gen.configs.models.dits import WanAnimate2Config, WanVideoConfig
from sglang.multimodal_gen.configs.models.encoders import (
BaseEncoderOutput,
CLIPVisionConfig,
@@ -45,6 +47,10 @@ def t5_postprocess_text(outputs: BaseEncoderOutput, _text_inputs) -> torch.Tenso
return prompt_embeds_tensor
+def animate2_postprocess_text(outputs: BaseEncoderOutput, text_inputs) -> torch.Tensor:
+ return outputs.last_hidden_state[0, : text_inputs["num_tokens"]]
+
+
@dataclass
class WanI2VCommonConfig(PipelineConfig):
# for all wan i2v pipelines
@@ -301,6 +307,127 @@ class Wan2_2_I2V_A14B_Config(WanI2V720PConfig):
)
+@dataclass
+class Wan_Animate_2_14B_Config(WanI2V720PConfig):
+ """Wan-Animate-2 14B (Wan2.2): reference image + reference video -> video.
+
+ Shares the Wan I2V component bundle (T5, CLIP vision, Wan VAE with the encoder
+ loaded) and swaps in the in-context ``WanAnimate2Config`` DiT; a single expert, so no
+ ``boundary_ratio`` switching. The bespoke stages read ``flow_shift`` from here
+ (carrying the audio track is a per-request sampling parameter); everything else below
+ restates the Wan2.2 14B defaults instead of inheriting the Wan2.1 ones from the base.
+ """
+
+ # reference image + reference video -> video is I2V-shaped.
+ task_type: ModelTaskType = ModelTaskType.I2V
+
+ # In-context animation DiT (in_channels=36 = 16 noise + 20 conditioning).
+ dit_config: DiTConfig = field(default_factory=WanAnimate2Config)
+
+ # Denoising stage. Used when building the sigma grid.
+ flow_shift: float | None = 5.0
+
+ postprocess_text_funcs: tuple[Callable, ...] = field(
+ default_factory=lambda: (animate2_postprocess_text,)
+ )
+
+ def tokenize_prompt(self, prompt, tokenizer, tok_kwargs):
+ text_inputs = super().tokenize_prompt(
+ prompt,
+ tokenizer,
+ {**tok_kwargs, "max_length": self.dit_config.arch_config.text_len},
+ )
+ # Count on the host before TextEncodingStage uploads inputs; no GPU scalar sync.
+ text_inputs["num_tokens"] = int(text_inputs["attention_mask"][0].gt(0).sum())
+ return text_inputs
+
+ # The component manager casts the VAE to this dtype at its first use (the reference
+ # encode in the before-denoising stage), so it is the precision the whole bespoke
+ # VAE path runs at; fp32 like the other Wan2.2 configs changes the output.
+ vae_decode_precision: str = "bf16"
+
+ def get_model_deployment_config(self) -> ModelDeploymentConfig:
+ # The Wan2.2 A14B modes (auto mode may layerwise-offload the 14B DiT) with the
+ # resident threshold the base had: at or above 60 GB free the DiT stays resident.
+ return ModelDeploymentConfig(
+ dit_layerwise_offload_modes=("auto", "memory"),
+ auto_dit_offload_prefetch_size=2,
+ keep_resident_min_available_gb=60,
+ keep_resident_components=("dit",),
+ )
+
+ def get_decode_scale_and_shift(
+ self,
+ device: torch.device | None,
+ dtype: torch.dtype | None,
+ vae: torch.nn.Module | None,
+ ) -> tuple[float, None]:
+ """``(1.0, None)``: ``WanAnimate2VaeAdapter.decode`` applies the latent scaling itself,
+ so the DecodingStage must not de-normalize again."""
+ return 1.0, None
+
+ def supports_disaggregation(self) -> bool:
+ # The stages share one WanAnimate2RequestState object (GPU tensors, a torch.Generator,
+ # the clip schedule) through batch.extra. The disaggregation transport forwards only
+ # JSON scalars and tensor pytrees (dict/list/tuple) from batch.extra and never the
+ # generator (disaggregation/extra_tensors.py, scheduler_mixin._EXCLUDE_FIELDS), so a
+ # split-role deployment would silently drop the state and fail at the first request.
+ return False
+
+ def validate_server_args(self, server_args: Any) -> None:
+ if server_args.use_fsdp_inference:
+ raise NotImplementedError(
+ "Wan-Animate-2 does not support --use-fsdp-inference; use "
+ "--dit-layerwise-offload or --tp-size instead."
+ )
+ # Ring splits the K/V across ranks; the in-context block mask indexes the full
+ # [generation | reference-video] K/V on one rank.
+ if server_args.ring_degree > 1:
+ raise NotImplementedError(
+ "Wan-Animate-2 does not support ring sequence parallelism "
+ f"(--ring-degree {server_args.ring_degree}); use --ulysses-degree N "
+ "with --ring-degree 1."
+ )
+ num_heads = self.dit_config.num_attention_heads
+ if num_heads % server_args.tp_size != 0:
+ raise ValueError(
+ f"Wan-Animate-2 tensor parallelism needs the {num_heads} attention "
+ f"heads divisible by --tp-size {server_args.tp_size}."
+ )
+ local_num_heads = num_heads // server_args.tp_size
+ if local_num_heads % server_args.ulysses_degree != 0:
+ raise ValueError(
+ f"Wan-Animate-2 Ulysses SP needs the TP-local head count ({local_num_heads}) "
+ f"divisible by --ulysses-degree {server_args.ulysses_degree}."
+ )
+ # The in-context self-attention runs on torch flex_attention whatever backend is
+ # selected; only the cross-attention and the reference pass would follow the flag.
+ requested_dit_backend = server_args.requested_component_attention_backend(
+ "transformer"
+ )
+ if (
+ server_args.is_arg_explicitly_set("attention_backend")
+ and server_args.attention_backend is not None
+ ) or requested_dit_backend is not None:
+ raise ValueError(
+ "Wan-Animate-2 does not support selecting the DiT attention backend "
+ f"(--attention-backend {server_args.attention_backend} / "
+ f"--component-attention-backends transformer={requested_dit_backend}): "
+ "its in-context self-attention runs on torch flex_attention only. Drop "
+ "the flag; the text encoder still takes "
+ "--component-attention-backends text_encoder=."
+ )
+ # The parent block applies the offline Q/K rotation inside its own attention
+ # path; forward_ref / forward_gen never do, so a rotated checkpoint would load
+ # and produce wrong output without a message.
+ if envs.SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION:
+ raise ValueError(
+ "Wan-Animate-2 does not support SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION "
+ "(offline Q/K rotation); unset it for this model."
+ )
+ super().validate_server_args(server_args)
+
+
# =============================================
# ============= Causal Self-Forcing =============
# =============================================
@@ -322,6 +449,7 @@ def register():
Wan2_2_I2V_A14B_SamplingParam,
Wan2_2_T2V_A14B_SamplingParam,
Wan2_2_TI2V_5B_SamplingParam,
+ Wan_Animate_2_14B_SamplingParam,
WanI2V_14B_480P_SamplingParam,
WanI2V_14B_720P_SamplingParam,
WanT2V_1_3B_SamplingParams,
@@ -423,3 +551,20 @@ def register():
"FastVideo/FastWan2.1-T2V-1.3B-Diffusers",
],
)
+ register_configs(
+ sampling_param_cls=Wan_Animate_2_14B_SamplingParam,
+ pipeline_config_cls=Wan_Animate_2_14B_Config,
+ hf_model_paths=[
+ "Wan-AI/Wan2.2-Animate-2-14B-Diffusers",
+ ],
+ model_detectors=[
+ # The registry calls this with the path and with model_index's lower-cased
+ # _class_name. Require the "-2-"/"_2" token so a v1 Wan2.2-Animate-14B
+ # checkpoint (a different model / pipeline) does not mis-route here.
+ lambda p: (
+ "wan_animate_2" in p.lower()
+ or "wan2.2-animate-2" in p.lower()
+ or p.lower() == "wananimate2pipeline"
+ )
+ ],
+ )
diff --git a/python/sglang/multimodal_gen/configs/sample/wan.py b/python/sglang/multimodal_gen/configs/sample/wan.py
index dfea73974d..a66976a590 100644
--- a/python/sglang/multimodal_gen/configs/sample/wan.py
+++ b/python/sglang/multimodal_gen/configs/sample/wan.py
@@ -1,10 +1,17 @@
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
# SPDX-License-Identifier: Apache-2.0
+import os
from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.sample.teacache import TeaCacheParams
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
def _wan_1_3b_coefficients(p: TeaCacheParams) -> list[float]:
@@ -267,6 +274,160 @@ class Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams):
)
+@dataclass
+class Wan_Animate_2_14B_SamplingParam(Wan2_2_Base_SamplingParams):
+ """Sampling defaults for Wan-Animate-2 14B."""
+
+ # Single expert: no guidance_scale_2 / boundary switching.
+ guidance_scale: float = 3.0
+ num_inference_steps: int = 40 # official README demo default
+
+ # The reference video is resampled to this rate and the output MP4 is written at it.
+ fps: int = 16
+ # Carry the reference video's audio track on the output, as the official pipeline does:
+ # the track is decoded in-process and the shared MP4 writer encodes it. False gives a
+ # silent video. A track that cannot be decoded is logged and the output stays silent.
+ enable_audio: bool = True
+
+ # Per-clip denoising length, 4k+1 (VAE stride 4); larger = better temporal coherence
+ # but more VRAM. Output length follows the reference video, not this.
+ clip_len: int = 37
+ # Text for the reference-video branch (prompt_ref_embeds). The official demo conditions it on a
+ # fixed description of the reference video, not on the character caption.
+ prompt_ref: str = "人物动作的参考视频"
+
+ # The reference image comes via the base image_path and the reference video via the
+ # base video_path; neither needs a model-specific field.
+
+ # (width, height) pairs; upstream Wan-Animate default is 640x800.
+ supported_resolutions: list[tuple[int, int]] | None = field(
+ default_factory=lambda: [
+ (640, 800), # upstream Wan-Animate default
+ (720, 1280), # 9:16
+ ]
+ )
+
+ def __post_init__(self) -> None:
+ # The in-context denoising loop runs every block at every step; the shared DiT's
+ # cache heuristics are never consulted.
+ if self.enable_teacache or self.enable_spectrum:
+ raise ValueError(
+ "Wan-Animate-2 does not support enable_teacache or enable_spectrum."
+ )
+ requested_clip_len = self.clip_len
+ self.clip_len = _round_clip_len(self.clip_len)
+ if self.clip_len < MIN_CLIP_LEN:
+ raise ValueError(
+ f"clip_len={requested_clip_len!r} is not supported: a clip is 4k+1 frames "
+ f"with k >= 1, so clip_len must be an int >= {MIN_CLIP_LEN}"
+ )
+ if self.clip_len != requested_clip_len:
+ logger.info(
+ "clip_len=%s is not 4k+1; using clip_len=%s",
+ requested_clip_len,
+ self.clip_len,
+ )
+ # The output length follows the reference video and the stages read clip_len, so
+ # `num_frames` is not a length control here; the multi-GPU frame alignment must
+ # leave it alone, and it must stay writable so the warmup probe can shrink.
+ self.adjust_frames = False
+ super().__post_init__()
+
+ @classmethod
+ def video_request_extra_fields(cls) -> frozenset[str]:
+ # Settable over the /v1/videos API: the per-clip length, the reference-branch
+ # text (the official gradio exposes prompt_ref as well) and the audio track. The
+ # reference video is the base video_path field, so it needs no entry here.
+ return frozenset({"clip_len", "prompt_ref", "enable_audio"})
+
+ def prepare_synthetic_warmup_request_for_queue(
+ self, req: Any, server_args: Any
+ ) -> None:
+ """Give the synthetic warmup request the driving video the generic builder does not
+ supply, sized so the request denoises exactly one clip."""
+ del server_args
+ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
+ single_clip_reference_video_len,
+ write_reference_video,
+ )
+
+ image_path = req.image_path
+ if isinstance(image_path, list):
+ image_path = image_path[0] if image_path else None
+ if not isinstance(image_path, str) or not image_path:
+ raise ValueError(
+ "Wan-Animate-2 synthetic warmup requires the warmup reference image"
+ )
+
+ # The builder sizes warmup by num_frames (the class default or its bounded cap), but
+ # the stage reads clip_len and the compiled attention is specialized per clip
+ # geometry, so warm up at the production clip length unless more frames were asked.
+ clip_len = _round_clip_len(max(int(req.num_frames), self.clip_len))
+ num_video_frames = single_clip_reference_video_len(clip_len)
+ fps = int(req.fps)
+
+ # Same directory and lifetime as the warmup image; the name carries the frame count
+ # because every warmup request is built before the first one runs.
+ video_path = os.path.join(
+ os.path.dirname(image_path),
+ f"warmup_driving_video_{num_video_frames}f_{fps}fps.mp4",
+ )
+ write_reference_video(
+ video_path,
+ _synthetic_driving_video_frames(num_video_frames),
+ fps=fps,
+ )
+
+ req.video_path = video_path
+ req.clip_len = clip_len
+ req.num_frames = clip_len
+ # The synthetic driving video has no audio track.
+ req.enable_audio = False
+
+ def _adjust_visual_fields(self, server_args: Any, pipeline_config: Any) -> None:
+ super()._adjust_visual_fields(server_args, pipeline_config)
+ if self.num_frames != type(self).num_frames:
+ logger.info(
+ "Wan-Animate-2 ignores num_frames=%s (the /v1/videos `seconds`): the output "
+ "length follows the reference video; clip_len=%s is the per-clip chunk",
+ self.num_frames,
+ self.clip_len,
+ )
+
+
+# Smallest clip: one conditioning frame plus one VAE temporal stride (4k+1, k >= 1).
+MIN_CLIP_LEN = 5
+
+
+def _round_clip_len(clip_len: int) -> int:
+ """Clips are 4k+1 pixel frames (Wan VAE temporal stride 4). Other values map to the
+ 4k+1 with the same k: 4k rounds up by one frame, 4k+2 and 4k+3 round down."""
+ if clip_len % 4 != 1:
+ clip_len = clip_len // 4 * 4 + 1
+ return clip_len
+
+
+# Pixel size of the synthetic warmup driving video; the stage rescales every frame to the
+# request area anyway, so this only has to be a size the shared video writer takes as is
+# and cheap to encode.
+_WARMUP_DRIVING_VIDEO_SIZE = 64
+
+
+def _synthetic_driving_video_frames(num_frames: int) -> np.ndarray:
+ """``[num_frames, 64, 64, 3]`` uint8 frames: a gradient that drifts one row per frame,
+ so consecutive frames differ like real motion."""
+ size = _WARMUP_DRIVING_VIDEO_SIZE
+ ramp = np.linspace(0, 255, size, dtype=np.float32)
+ rows = np.broadcast_to(ramp[:, None], (size, size))
+ cols = np.broadcast_to(ramp[None, :], (size, size))
+ frames = np.empty((num_frames, size, size, 3), dtype=np.uint8)
+ for index in range(num_frames):
+ frames[index, ..., 0] = np.roll(rows, index, axis=0)
+ frames[index, ..., 1] = cols
+ frames[index, ..., 2] = 127
+ return frames
+
+
@dataclass
class Turbo_Wan2_2_I2V_A14B_SamplingParam(Wan2_2_Base_SamplingParams):
guidance_scale: float = 3.5 # high_noise
diff --git a/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py b/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py
index 7901846cb0..d9c20d19c1 100644
--- a/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py
+++ b/python/sglang/multimodal_gen/runtime/loader/component_loaders/component_loader.py
@@ -710,6 +710,9 @@ class WeightOverrideComponentLoader(ComponentLoader):
"""Base for loaders that consume an exact weights-only override."""
ignored_checkpoint_prefixes: tuple[str, ...] = ()
+ # True: a checkpoint key that maps to no model parameter fails the load instead
+ # of being logged and skipped.
+ strict_checkpoint_keys: bool = False
def load_state_dict_model(
self,
@@ -738,7 +741,7 @@ class WeightOverrideComponentLoader(ComponentLoader):
fsdp_inference=server_args.should_use_fsdp_for_component(component_name),
param_dtype=dtype,
reduce_dtype=torch.float32,
- strict=False,
+ strict=self.strict_checkpoint_keys,
weight_load_plan=weight_load_plan,
checkpoint_key_filter=checkpoint_key_filter,
weights_iterator=weights_iterator,
diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
index bbf38d7a53..11daed7066 100644
--- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
+++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py
@@ -214,22 +214,28 @@ def fit_auto_residency_probe(
floor_units = min((record.workload_units() for record in records), default=0)
fitted, steps = req, 0
while True:
- units = (
- max(1, int(fitted.width or 1))
- * max(1, int(fitted.height or 1))
- * max(1, int(fitted.num_frames or 1))
- )
+ units = _probe_workload_units(fitted)
estimate = estimate_default_workload_peak_bytes(
records=records, target_units=units
)
if estimate is None or estimate <= budget or units <= floor_units:
return fitted, estimate, steps
lighter = lighten_warmup_req(server_args, fitted)
- if lighter is None:
+ # Sampling params that pin the frame count (Wan-Animate-2 mirrors clip_len
+ # in __post_init__) hand back an equal-size req; treat that as the floor.
+ if lighter is None or _probe_workload_units(lighter) >= units:
return fitted, estimate, steps
fitted, steps = lighter, steps + 1
+def _probe_workload_units(req: Req) -> int:
+ return (
+ max(1, int(req.width or 1))
+ * max(1, int(req.height or 1))
+ * max(1, int(req.num_frames or 1))
+ )
+
+
class GPUWorker(GPUWorkerPostTrainingMixin):
"""
A worker that executes the model on a single GPU.
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py
new file mode 100644
index 0000000000..9e55ca5ab1
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2.py
@@ -0,0 +1,445 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Wan-Animate-2 DiT. Subclasses WanTransformer3DModel so TP / Ulysses SP shard it
+# unchanged; only the per-block self-attention is swapped for the in-context mechanism
+# (build_reference_kv runs forward_ref once per clip and returns each block's rotated reference-video
+# K/V, forward_gen attends over [gen tokens, then that reference-video K/V]). No extra parameters.
+# Names kept from the reference implementation: the forward_ref / forward_gen methods and the
+# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image.
+from collections import OrderedDict
+from typing import Any
+
+import torch
+import torch.nn as nn
+from torch.nn.attention.flex_attention import BlockMask, create_block_mask
+
+from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
+ WanAnimate2Config,
+)
+from sglang.multimodal_gen.runtime.distributed import get_sp_world_size
+from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
+ gather_seq,
+ shard_seq,
+)
+from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
+ QuantizationConfig,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
+ WanAnimate2TransformerBlock,
+ _InContextLayout,
+ _reference_video_key_mask,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
+ WanAnimate2ClipConditioning,
+ WanAnimate2ReferenceKV,
+)
+from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformer3DModel
+from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
+
+
+class WanAnimate2Transformer3DModel(WanTransformer3DModel):
+ """Wan-Animate-2 DiT (in-context variant) on the Wan2.2-I2V-14B backbone.
+
+ Attention goes through USPAttention, so callers must wrap the forward in
+ ``set_forward_context(...)``.
+ """
+
+ # Registry alias: the diffusers class name of this transformer.
+ _aliases = ["WanAnimate2Transformer3DModel"]
+
+ param_names_mapping = WanAnimate2Config().param_names_mapping
+ reverse_param_names_mapping = WanAnimate2Config().reverse_param_names_mapping
+ lora_param_names_mapping = WanAnimate2Config().lora_param_names_mapping
+
+ def __init__(
+ self,
+ config: WanAnimate2Config,
+ hf_config: dict[str, Any] | None = None,
+ quant_config: QuantizationConfig | None = None,
+ ) -> None:
+ super().__init__(
+ config=config, hf_config=hf_config or {}, quant_config=quant_config
+ )
+
+ # RoPE placement of the reference-video tokens relative to the gen grid; plain
+ # ints from the config, not parameters, so the Wan state dict is unchanged.
+ self.refer_offset_t = config.refer_offset_t
+ self.refer_offset_h = config.refer_offset_h
+ # <0: place reference-video tokens right of the gen grid (w offset = gen W in tokens).
+ self.refer_offset_w = config.refer_offset_w
+ self.refer_stride = config.refer_stride
+ # Prompt embeddings are padded/truncated to this many tokens.
+ self.max_text_len = config.text_len
+
+ self._block_masks: OrderedDict[tuple[int, int, int, str], BlockMask] = (
+ OrderedDict()
+ )
+ # RoPE tables keyed by the (f, h, w) position ranges and device; see _rope_tables.
+ self._rope_cache: OrderedDict[tuple, tuple[torch.Tensor, torch.Tensor]] = (
+ OrderedDict()
+ )
+
+ def _make_block(
+ self,
+ *,
+ index: int,
+ config: WanAnimate2Config,
+ quant_config: QuantizationConfig | None,
+ ) -> nn.Module:
+ # Same constructor args as the Wan block and no new params, so the Wan checkpoint
+ # mapping and TP sharding apply unchanged.
+ return WanAnimate2TransformerBlock(
+ config.num_attention_heads * config.attention_head_dim,
+ config.ffn_dim,
+ config.num_attention_heads,
+ config.qk_norm,
+ config.cross_attn_norm,
+ config.eps,
+ config.added_kv_proj_dim,
+ self._supported_attention_backends
+ | {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
+ prefix=f"blocks.{index}",
+ attention_type=config.attention_type,
+ sla_topk=config.sla_topk,
+ quant_config=quant_config,
+ log_scale=config.log_scale,
+ )
+
+ # Device and dtype of the resident weights; the patch embedding is never offloaded.
+ @property
+ def _device(self) -> torch.device:
+ return self.patch_embedding.proj.weight.device
+
+ @property
+ def _dtype(self) -> torch.dtype:
+ return self.patch_embedding.proj.weight.dtype
+
+ def _rope_tables(
+ self,
+ f_range: tuple[int, int, int], # (start, length, stride)
+ h_range: tuple[int, int], # (start, length)
+ w_range: tuple[int, int],
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """RoPE ``(cos, sin)``, each ``[f*h*w, D//2]`` fp32, for the full grid.
+
+ Row i holds the position of token i, flattened in (f, h, w) row-major order
+ like the patch embedding output.
+
+ The table is cached per ranges."""
+ key = (f_range, h_range, w_range, str(self._device))
+ if key in self._rope_cache:
+ self._rope_cache.move_to_end(key)
+ return self._rope_cache[key]
+ # Bound retained GPU tables; evict before allocating the next geometry.
+ if len(self._rope_cache) >= 4:
+ self._rope_cache.popitem(last=False)
+ f_start, f_len, f_stride = f_range
+ f_idx = torch.arange(f_len, device=self._device) * f_stride + f_start
+ h_idx = torch.arange(h_range[1], device=self._device) + h_range[0]
+ w_idx = torch.arange(w_range[1], device=self._device) + w_range[0]
+ # Positions are built explicitly (not via the rank-sharding forward_from_grid) because
+ # the in-context blocks all-to-all-gather the full sequence before applying RoPE.
+ ff, hh, ww = torch.meshgrid(f_idx, h_idx, w_idx, indexing="ij")
+ positions = torch.stack([ff.reshape(-1), hh.reshape(-1), ww.reshape(-1)], dim=1)
+ cos, sin = self.rotary_emb.forward_uncached(positions.long())
+ tables = (cos.float(), sin.float())
+ self._rope_cache[key] = tables
+ return tables
+
+ def _generation_video_rope(
+ self, f: int, h: int, w: int
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """RoPE tables for the generated video's ``f x h x w`` token grid; positions start at 0."""
+ return self._rope_tables((0, f, 1), (0, h), (0, w))
+
+ def _reference_video_rope(
+ self, f: int, h: int, w: int, generation_video_w: int
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """RoPE tables for the reference video's ``f x h x w`` token grid, shifted by the
+ ``refer_*`` offsets. When ``refer_offset_w`` is -1, the reference-video grid is placed
+ directly to the right of the generated grid (width offset = ``generation_video_w``)."""
+ offset_w = (
+ generation_video_w if self.refer_offset_w == -1 else self.refer_offset_w
+ )
+ return self._rope_tables(
+ (self.refer_offset_t, f, self.refer_stride),
+ (self.refer_offset_h, h),
+ (offset_w, w),
+ )
+
+ def _get_block_mask_from_layout(self, layout: _InContextLayout) -> BlockMask:
+ """Cached flex_attention ``BlockMask`` over the padded [gen | reference-video] layout."""
+ key = (
+ layout.max_generation_video_num_tokens,
+ layout.max_reference_video_num_tokens,
+ layout.max_tokens_per_frame,
+ str(self._device),
+ )
+ if key in self._block_masks:
+ self._block_masks.move_to_end(key)
+ return self._block_masks[key]
+ if len(self._block_masks) >= 2:
+ self._block_masks.popitem(last=False)
+
+ # _compile=True keeps the block-sparse layout consistent with the compiled
+ # flex_attention; built once per key, so the compile cost is paid once.
+ block_mask = create_block_mask(
+ layout.block_mask_fn(),
+ B=None,
+ H=None,
+ Q_LEN=layout.padded_generation_video_len,
+ KV_LEN=layout.padded_kv_len,
+ device=self._device,
+ _compile=True,
+ )
+ self._block_masks[key] = block_mask
+ return block_mask
+
+ def _patch_embeddings(
+ self,
+ latent_36ch: torch.Tensor, # [B, 36, f, H, W]
+ ) -> tuple[torch.Tensor, tuple[int, int, int]]:
+ """Patch-embed the latent into the token stream ``[B, f*h*w, C]`` and return the
+ token grid_size ``(f, h, w)``; tokens are flattened row-major, w fastest."""
+ p_t, p_h, p_w = self.patch_size
+ _, _, num_frames, height, width = latent_36ch.shape
+ grid_size = (num_frames // p_t, height // p_h, width // p_w)
+ hidden_states = self.patch_embedding(latent_36ch)
+ hidden_states = hidden_states.flatten(2).transpose(1, 2).contiguous()
+ return hidden_states, grid_size
+
+ def _time_embeddings(
+ self, timestep: torch.Tensor
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ timestep_embeddings = self.condition_embedder.time_embedder(timestep, None)
+ timestep_modulation = self.condition_embedder.time_modulation(
+ timestep_embeddings
+ ).unflatten(1, (6, -1))
+ return timestep_embeddings, timestep_modulation
+
+ def prepare_context(
+ self,
+ prompt_embeddings: torch.Tensor,
+ image_embeddings: torch.Tensor | None,
+ ) -> torch.Tensor:
+ """Project one branch's timestep-invariant context; owned by the current clip."""
+ prompt_embeddings = self._pad_or_truncate_prompt_embeddings(prompt_embeddings)
+ context = self.condition_embedder.text_embedder(prompt_embeddings)
+ if image_embeddings is not None:
+ image_context = self.condition_embedder.image_embedder(image_embeddings)
+ context = torch.cat([image_context, context], dim=1)
+ return context.to(self._dtype)
+
+ def _pad_or_truncate_prompt_embeddings(
+ self, prompt_embeddings: torch.Tensor
+ ) -> torch.Tensor:
+ """Zero-pad or truncate the prompt embeddings (``[L, C]`` or ``[B, L, C]``) to
+ ``[B, max_text_len, C]``, as the official model does; cross-attention attends the pad rows too."""
+ if prompt_embeddings.dim() == 2:
+ prompt_embeddings = prompt_embeddings.unsqueeze(0) # [L, C] -> [1, L, C]
+
+ batch_size, num_tokens, channels = prompt_embeddings.shape
+ if num_tokens < self.max_text_len:
+ pad = prompt_embeddings.new_zeros(
+ batch_size, self.max_text_len - num_tokens, channels
+ )
+ prompt_embeddings = torch.cat([prompt_embeddings, pad], dim=1)
+ elif num_tokens > self.max_text_len:
+ prompt_embeddings = prompt_embeddings[:, : self.max_text_len]
+ return prompt_embeddings.to(self._dtype)
+
+ @torch.no_grad()
+ def build_reference_kv(
+ self, *, clip_cond: WanAnimate2ClipConditioning
+ ) -> WanAnimate2ReferenceKV:
+ """One forward_ref pass over the clip's reference video, returning every block's rotated K and V.
+
+ Text- and timestep-independent, so one result serves every step and CFG branch of the
+ clip; the caller owns it and the DiT keeps no copy. Attention goes through USPAttention,
+ so callers must wrap this in ``set_forward_context(...)`` like ``forward``.
+ """
+
+ # 36-channel reference-video input: fp32 latents (16) over the bf16 condition (20),
+ # both cast to the param dtype so the cat runs in bf16 (see _make_generation_input).
+ reference_video_latent_36ch = torch.cat(
+ [
+ clip_cond.reference_video_latents.to(self._dtype),
+ clip_cond.reference_video_condition.to(self._dtype),
+ ],
+ dim=0,
+ ).unsqueeze(0) # [1, 36, cf, H', W']
+ reference_video_frame_0_image_embeddings = (
+ clip_cond.reference_video_frame_0_image_embeddings
+ ) # [1, 257, 1280] bf16
+
+ # reference-video timestep is forced to 1.
+ reference_video_timestep = torch.ones(1, device=self._device, dtype=torch.long)
+ hidden_states, (reference_video_f, reference_video_h, reference_video_w) = (
+ self._patch_embeddings(reference_video_latent_36ch)
+ )
+ _, timestep_modulation = self._time_embeddings(reference_video_timestep)
+ reference_video_encoder_hidden_states = self.prepare_context(
+ clip_cond.prompt_ref_embeddings, reference_video_frame_0_image_embeddings
+ )
+
+ reference_video_rope_cos, reference_video_rope_sin = self._reference_video_rope(
+ reference_video_f,
+ reference_video_h,
+ reference_video_w,
+ generation_video_w=clip_cond.generation_video_grid_sizes[2],
+ )
+ reference_video_num_tokens = (
+ reference_video_f * reference_video_h * reference_video_w
+ )
+
+ # Ulysses SP: shard the reference-video sequence; each block a2a-gathers it and caches
+ # full-seq / head-shard K/V. The reference-video hidden states are discarded, so no gather.
+ # The gathered sequence is tail-padded to a multiple of the SP size; one key mask over
+ # that padding serves every block (None when nothing is padded).
+ reference_video_key_mask = None
+ sp_size = get_sp_world_size()
+ if sp_size > 1:
+ hidden_states, _ = shard_seq(hidden_states, dim=1)
+ gathered_seq_len = hidden_states.shape[1] * sp_size
+ if gathered_seq_len != reference_video_num_tokens:
+ reference_video_key_mask = _reference_video_key_mask(
+ batch_size=hidden_states.shape[0],
+ seq_len=gathered_seq_len,
+ num_valid=reference_video_num_tokens,
+ device=hidden_states.device,
+ )
+
+ k_by_block: dict[int, torch.Tensor] = {}
+ v_by_block: dict[int, torch.Tensor] = {}
+ # Call via __call__ (not forward_ref) so the layerwise-offload hooks fire.
+ for idx, block in enumerate(self.blocks):
+ hidden_states = block(
+ kv_cache_mode="ref",
+ hidden_states=hidden_states,
+ encoder_hidden_states=reference_video_encoder_hidden_states,
+ timestep_modulation=timestep_modulation,
+ reference_video_rope_cos=reference_video_rope_cos,
+ reference_video_rope_sin=reference_video_rope_sin,
+ reference_video_num_tokens=reference_video_num_tokens,
+ index=idx,
+ k_cache=k_by_block,
+ v_cache=v_by_block,
+ reference_video_key_mask=reference_video_key_mask,
+ )
+ return WanAnimate2ReferenceKV(k=k_by_block, v=v_by_block)
+
+ @torch.no_grad()
+ def forward(
+ self,
+ hidden_states: torch.Tensor, # [16, f, h, w] gen latent (B=1)
+ encoder_hidden_states: torch.Tensor, # this CFG branch's prompt embeddings [L, C]
+ timestep: torch.Tensor, # [1]
+ # [1, 257, 1280] bf16 CLIP embed of the reference image
+ encoder_hidden_states_image: torch.Tensor | None = None,
+ guidance: torch.Tensor | None = None,
+ *,
+ # This clip's K/V from build_reference_kv; the same object for every step and CFG branch.
+ reference_kv: WanAnimate2ReferenceKV,
+ # This clip's conditioning from build_clip_conditioning.
+ clip_cond: WanAnimate2ClipConditioning,
+ projected_context: torch.Tensor | None = None,
+ # The unconditional CFG branch skips block 9 (see _run_generation_blocks).
+ is_unconditional: bool = False,
+ ) -> torch.Tensor:
+ sp_size = get_sp_world_size()
+
+ latent_36ch = self._make_generation_input(
+ hidden_states, clip_cond.generation_condition
+ )
+ hidden_states, (f, h, w) = self._patch_embeddings(latent_36ch)
+ timestep_embeddings, timestep_modulation = self._time_embeddings(timestep)
+ generation_encoder_hidden_states = (
+ projected_context
+ if projected_context is not None
+ else self.prepare_context(
+ encoder_hidden_states, encoder_hidden_states_image
+ )
+ )
+ layout = _InContextLayout(
+ (f, h, w),
+ clip_cond.reference_video_grid_sizes,
+ clip_cond.full_clip_grid_sizes,
+ )
+
+ # No seq padding here: hidden_states is exactly f*h*w tokens and the in-context
+ # layout pads internally. Under Ulysses SP each block a2a-gathers the full
+ # sequence, so shard tail padding lands at the global tail where forward_gen
+ # expects it.
+ if sp_size > 1:
+ hidden_states, _ = shard_seq(hidden_states, dim=1)
+ hidden_states = self._run_generation_blocks(
+ hidden_states,
+ encoder_hidden_states=generation_encoder_hidden_states,
+ timestep_modulation=timestep_modulation,
+ layout=layout,
+ reference_kv=reference_kv,
+ is_unconditional=is_unconditional,
+ )
+ if sp_size > 1:
+ # Gather the shards and trim tail padding back to the clip's token count.
+ hidden_states = gather_seq(
+ hidden_states, layout.generation_video_num_tokens, dim=1
+ )
+
+ return self._unpatchify(hidden_states, timestep_embeddings, (f, h, w))
+
+ def _make_generation_input(
+ self,
+ latent: torch.Tensor, # [16, f, h, w] fp32
+ generation_condition: torch.Tensor, # [20, f, h, w] bf16
+ ) -> torch.Tensor:
+ """36-channel DiT input ``[1, 36, f, h, w]``: the fp32 noisy latent over the bf16 generation condition."""
+ # Both cast to the param dtype so the cat runs in bf16 and does not rely on autocast;
+ # the fp32 latent would be rounded to bf16 by the patch embedding anyway.
+ return torch.cat(
+ [latent.to(self._dtype), generation_condition.to(self._dtype)], dim=0
+ ).unsqueeze(0)
+
+ def _run_generation_blocks(
+ self,
+ hidden_states: torch.Tensor,
+ *,
+ encoder_hidden_states: torch.Tensor,
+ timestep_modulation: torch.Tensor,
+ layout: _InContextLayout,
+ reference_kv: WanAnimate2ReferenceKV,
+ is_unconditional: bool,
+ ) -> torch.Tensor:
+ """forward_gen through every block with the in-context RoPE tables and block mask."""
+ generation_video_rope_cos, generation_video_rope_sin = (
+ self._generation_video_rope(
+ layout.generation_video_f,
+ layout.generation_video_h,
+ layout.generation_video_w,
+ )
+ )
+ block_mask = self._get_block_mask_from_layout(layout)
+
+ # Call via __call__ (not forward_gen) so the layerwise-offload hooks fire.
+ for idx, block in enumerate(self.blocks):
+ # Block 9 is dropped on the unconditional branch, as in the reference
+ # wan_animate_2_model.forward_gen; the authors give no rationale.
+ if is_unconditional and idx == 9:
+ continue
+ hidden_states = block(
+ kv_cache_mode="gen",
+ hidden_states=hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ timestep_modulation=timestep_modulation,
+ gen_cos=generation_video_rope_cos,
+ gen_sin=generation_video_rope_sin,
+ block_mask=block_mask,
+ layout=layout,
+ index=idx,
+ k_cache=reference_kv.k,
+ v_cache=reference_kv.v,
+ )
+ return hidden_states
+
+
+EntryClass = WanAnimate2Transformer3DModel
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py
new file mode 100644
index 0000000000..2c906f3076
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_block.py
@@ -0,0 +1,532 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Wan-Animate-2 DiT block: in-context self-attention over [gen tokens, then cached
+# reference-video K/V], its RoPE / Ulysses helpers, and the per-clip _InContextLayout
+# geometry.
+# Names kept from the reference implementation: the forward_ref / forward_gen methods and the
+# kv_cache_mode values "ref" / "gen"; there "ref" means the reference video (the motion source), not the reference image.
+from __future__ import annotations
+
+import math
+from collections.abc import Callable
+from functools import lru_cache, partial
+from typing import Any, Literal
+
+import torch
+from torch.nn.attention.flex_attention import BlockMask, flex_attention
+
+from sglang.multimodal_gen.runtime.distributed import (
+ get_sp_world_size,
+)
+from sglang.multimodal_gen.runtime.layers.usp import (
+ _usp_input_all_to_all_qkv as _sp_gather_qkv,
+)
+from sglang.multimodal_gen.runtime.layers.usp import (
+ _usp_output_all_to_all,
+)
+from sglang.multimodal_gen.runtime.models.dits.wanvideo import WanTransformerBlock
+
+# Eager flex_attention materializes the full [Q, KV] scores over the padded [gen | reference-video]
+# sequence (tens of GiB at video resolution); compiled, it is one fused kernel.
+_flex_attention_compiled = torch.compile(
+ flex_attention,
+ dynamic=False,
+ mode="max-autotune",
+ fullgraph=True,
+ backend="inductor",
+)
+
+
+def _apply_rope_interleaved(
+ x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
+) -> torch.Tensor:
+ """Interleaved (GPT-J style) RoPE, identical to ``_apply_rotary_emb(is_neox_style=False)``;
+ ``x`` is ``[B, S, H, D]``, ``cos`` / ``sin`` are ``[S, D//2]`` fp32."""
+ cos = cos[None, :, None, :] # [1, S, 1, D//2]
+ sin = sin[None, :, None, :]
+ x1 = x[..., 0::2].float()
+ x2 = x[..., 1::2].float()
+ o1 = x1 * cos - x2 * sin
+ o2 = x2 * cos + x1 * sin
+ out = torch.stack([o1, o2], dim=-1).flatten(-2)
+ return out.type_as(x)
+
+
+def _score_mod_impl(
+ score: torch.Tensor,
+ b_idx: torch.Tensor,
+ h_idx: torch.Tensor,
+ q_idx: torch.Tensor,
+ kv_idx: torch.Tensor,
+ hw: int,
+ log_scale: float,
+) -> torch.Tensor:
+ # Biases the keys of generation frame 1, [hw, 2*hw) in the padded layout, exactly as the
+ # official Wan-Animate-2 code does; non-zero only when the checkpoint config sets log_scale.
+ condition = (kv_idx >= hw) & (kv_idx < 2 * hw)
+ return torch.where(condition, score + log_scale, score)
+
+
+@lru_cache(maxsize=None)
+def _make_score_mod(hw: int, log_scale: float) -> Callable[..., torch.Tensor]:
+ # Stable object identity per (hw, log_scale): compiled flex_attention guards on the
+ # score_mod object, so a fresh closure per call would recompile every clip.
+ return partial(_score_mod_impl, hw=hw, log_scale=log_scale)
+
+
+def _sp_scatter_out(attn_output: torch.Tensor) -> torch.Tensor:
+ """Ulysses output a2a: [B, S_full, N_local, C] -> [B, S_local, N_tp, C]."""
+ return _usp_output_all_to_all(attn_output, head_dim=2)
+
+
+def _reference_video_key_mask(
+ *, batch_size: int, seq_len: int, num_valid: int, device: torch.device
+) -> torch.Tensor:
+ """``[B, S]`` bool key mask: True for the leading ``num_valid`` reference-video tokens."""
+ key_mask = torch.zeros((batch_size, seq_len), dtype=torch.bool, device=device)
+ key_mask[:, :num_valid] = True
+ return key_mask
+
+
+class WanAnimate2TransformerBlock(WanTransformerBlock):
+ """Wan block with the Wan-Animate-2 in-context self-attention.
+
+ No new parameters: forward_ref / forward_gen reuse the inherited modules and only
+ route the self-attention differently.
+ """
+
+ def __init__(self, *args: Any, log_scale: float = 0.0, **kwargs: Any) -> None:
+ super().__init__(*args, **kwargs)
+ if self.use_offline_qk_rotation:
+ raise ValueError(
+ "Wan-Animate-2 blocks do not apply the offline Q/K rotation that "
+ "SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION selects for this checkpoint."
+ )
+ # Additive score bias on the keys of generation frame 1 (_score_mod_impl); 0.0 skips it.
+ self.log_scale = log_scale
+
+ def _modulate(
+ self, timestep_modulation: torch.Tensor
+ ) -> tuple[
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ ]:
+ """14B modulation split of ``timestep_modulation`` ``[B, 6, C]`` into six ``[B, 1, C]`` fp32 chunks; mirrors wanvideo.py."""
+ assert timestep_modulation.dim() == 3, (
+ "WanAnimate2TransformerBlock expects the 14B timestep modulation [B, 6, inner]; the ti2v "
+ "seq-len modulation path is not used by Wan-Animate-2."
+ )
+ e = self.scale_shift_table + timestep_modulation.float()
+ shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa = e.chunk(
+ 6, dim=1
+ )
+ return shift_msa, scale_msa, gate_msa, c_shift_msa, c_scale_msa, c_gate_msa
+
+ def forward(
+ self, *, kv_cache_mode: Literal["ref", "gen"], **kwargs: Any
+ ) -> torch.Tensor:
+ """Dispatch to forward_ref / forward_gen through ``nn.Module.__call__`` so the
+ layerwise-offload module hooks fire for both passes; keyword arguments only."""
+ if kv_cache_mode == "ref":
+ return self.forward_ref(**kwargs)
+ if kv_cache_mode == "gen":
+ return self.forward_gen(**kwargs)
+ raise ValueError(
+ f"WanAnimate2TransformerBlock: unknown kv_cache_mode {kv_cache_mode!r} "
+ "(expected 'ref' or 'gen')."
+ )
+
+ def forward_ref(
+ self,
+ *,
+ hidden_states: torch.Tensor, # [B, S_reference_video, C] reference-video latent stream
+ encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] reference-video context (CLIP tokens, then text)
+ timestep_modulation: torch.Tensor, # [B, 6, C], reference-video timestep forced to 1
+ reference_video_rope_cos: torch.Tensor, # reference-video RoPE cos [reference_video_num_tokens, D//2]
+ reference_video_rope_sin: torch.Tensor,
+ reference_video_num_tokens: int, # reference_video_f * reference_video_h * reference_video_w (valid reference-video tokens)
+ index: int,
+ k_cache: dict[int, torch.Tensor],
+ v_cache: dict[int, torch.Tensor],
+ # [B, S_reference_video] key mask for the SP tail padding, built once per clip by the
+ # caller; None when the gathered sequence has no padding.
+ reference_video_key_mask: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ """Cache this block's rotated reference-video K and V, then run the reference-video self-attention
+ (reference-video RoPE, pad keys masked) and return the updated reference-video stream."""
+ orig_dtype = hidden_states.dtype
+ shift, scale, gate, c_shift, c_scale, c_gate = self._modulate(
+ timestep_modulation
+ )
+
+ norm_hidden_states = self.norm1(hidden_states, shift, scale)
+ query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
+
+ # Ulysses SP: gather the full reference-video sequence / shard heads before caching, so the
+ # cache matches forward_gen's post-a2a layout.
+ sp_size = get_sp_world_size()
+ if sp_size > 1:
+ query, key, value = _sp_gather_qkv(query, key, value)
+
+ # Reference-video self-attention. Under SP, S_reference_video is the gathered
+ # (tail-padded) length: RoPE the valid slice and mask the pad keys. Without padding the
+ # whole sequence is valid and no mask is passed: USPAttention serves a dense key mask
+ # through SDPA instead of the configured backend.
+ reference_video_seq_len = query.shape[1]
+ if reference_video_seq_len == reference_video_num_tokens:
+ q_roped = _apply_rope_interleaved(
+ query, reference_video_rope_cos, reference_video_rope_sin
+ )
+ k_roped = _apply_rope_interleaved(
+ key, reference_video_rope_cos, reference_video_rope_sin
+ )
+ key_mask = None
+ else:
+ q_roped = query.clone()
+ k_roped = key.clone()
+ q_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved(
+ query[:, :reference_video_num_tokens],
+ reference_video_rope_cos,
+ reference_video_rope_sin,
+ )
+ k_roped[:, :reference_video_num_tokens] = _apply_rope_interleaved(
+ key[:, :reference_video_num_tokens],
+ reference_video_rope_cos,
+ reference_video_rope_sin,
+ )
+ key_mask = reference_video_key_mask
+ if key_mask is None:
+ key_mask = _reference_video_key_mask(
+ batch_size=hidden_states.shape[0],
+ seq_len=reference_video_seq_len,
+ num_valid=reference_video_num_tokens,
+ device=hidden_states.device,
+ )
+ # Reference positions are invariant within a clip. Keep the rotated key in the
+ # original cache storage; SP's value can share the gathered QKV allocation.
+ k_cache[index] = key.copy_(k_roped)
+ v_cache[index] = value
+ if sp_size > 1:
+ # q/k/v are already gathered: skip USPAttention's own a2a, then scatter back.
+ attn_output = self.attn1(
+ q_roped,
+ k_roped,
+ value,
+ attn_mask=key_mask,
+ skip_sequence_parallel_override=True,
+ )
+ attn_output = _sp_scatter_out(attn_output)
+ else:
+ attn_output = self.attn1(q_roped, k_roped, value, attn_mask=key_mask)
+ attn_output = attn_output.flatten(2)
+
+ return self._cross_and_ffn(
+ hidden_states,
+ attn_output,
+ encoder_hidden_states,
+ gate,
+ c_shift,
+ c_scale,
+ c_gate,
+ orig_dtype,
+ )
+
+ def forward_gen(
+ self,
+ *,
+ hidden_states: torch.Tensor, # [B, seq_len, C] gen stream (padded)
+ encoder_hidden_states: torch.Tensor, # [B, 257 + L, C] gen context (CLIP tokens, then text)
+ timestep_modulation: torch.Tensor, # [B, 6, C]
+ gen_cos: torch.Tensor, # gen-RoPE cos [generation_video_num_tokens, D//2]
+ gen_sin: torch.Tensor,
+ block_mask: BlockMask, # per full-length clip grid
+ layout: _InContextLayout,
+ index: int,
+ k_cache: dict[int, torch.Tensor],
+ v_cache: dict[int, torch.Tensor],
+ ) -> torch.Tensor:
+ """One generation block with in-context masked self-attention over
+ [gen tokens, then cached reference-video]."""
+ orig_dtype = hidden_states.dtype
+ shift, scale, gate, c_shift, c_scale, c_gate = self._modulate(
+ timestep_modulation
+ )
+
+ norm_hidden_states = self.norm1(hidden_states, shift, scale)
+ query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
+
+ # Ulysses SP: gather the full gen sequence / shard heads. The cached reference-video K/V are
+ # already in this layout.
+ sp_size = get_sp_world_size()
+ if sp_size > 1:
+ query, key, value = _sp_gather_qkv(query, key, value)
+
+ # gen-RoPE on the valid gen tokens only (padding kept unrotated).
+ num_generation_tokens = layout.generation_video_num_tokens
+ q_roped = _apply_rope_interleaved(
+ query[:, :num_generation_tokens], gen_cos, gen_sin
+ )
+ k_roped = _apply_rope_interleaved(
+ key[:, :num_generation_tokens], gen_cos, gen_sin
+ )
+ if query.shape[1] != num_generation_tokens:
+ q_roped = torch.cat([q_roped, query[:, num_generation_tokens:]], dim=1)
+ k_roped = torch.cat([k_roped, key[:, num_generation_tokens:]], dim=1)
+
+ # Reference-video K is already rotated by forward_ref, including SP tail padding.
+ reference_video_k = k_cache[index]
+ reference_video_v = v_cache[index]
+
+ attn_output = self._in_context_attention(
+ q_roped,
+ k_roped,
+ value,
+ reference_video_k,
+ reference_video_v,
+ block_mask,
+ layout,
+ )
+ if sp_size > 1:
+ # scatter the sequence back / gather heads.
+ attn_output = _sp_scatter_out(attn_output)
+ attn_output = attn_output.flatten(2)
+
+ return self._cross_and_ffn(
+ hidden_states,
+ attn_output,
+ encoder_hidden_states,
+ gate,
+ c_shift,
+ c_scale,
+ c_gate,
+ orig_dtype,
+ )
+
+ def _in_context_attention(
+ self,
+ q: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd)
+ k: torch.Tensor, # [B, seq_len, H, D] (gen-RoPE'd)
+ v: torch.Tensor, # [B, seq_len, H, D]
+ reference_video_k: torch.Tensor, # [B, reference_video_seq_len, H, D] (reference-video RoPE'd)
+ reference_video_v: torch.Tensor, # [B, reference_video_seq_len, H, D]
+ block_mask: BlockMask,
+ layout: _InContextLayout,
+ ) -> torch.Tensor:
+ """Build the padded [gen | reference-video] layout and run compiled flex_attention.
+
+ The block-mask / score_mod are over sequence positions only, so TP composes
+ trivially, and under Ulysses SP the callers hand in the full sequence with sharded
+ heads, so the layout and mask are identical to sp=1."""
+ batch_size, _, num_heads, head_dim = q.shape
+ device, dtype = q.device, q.dtype
+ padded_generation_video_len = layout.padded_generation_video_len
+ padded_kv_len = layout.padded_kv_len
+
+ q_padding = q[:, layout.generation_video_num_tokens :]
+
+ q_padded = torch.zeros(
+ batch_size,
+ padded_generation_video_len,
+ num_heads,
+ head_dim,
+ device=device,
+ dtype=dtype,
+ )
+ k_padded = torch.zeros(
+ batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype
+ )
+ v_padded = torch.zeros(
+ batch_size, padded_kv_len, num_heads, head_dim, device=device, dtype=dtype
+ )
+
+ num_latent_frames = layout.generation_video_f
+ latent_hw = layout.generation_video_tokens_per_frame
+ q_valid = q[:, : layout.generation_video_num_tokens].view(
+ batch_size, num_latent_frames, latent_hw, num_heads, head_dim
+ )
+ k_valid = k[:, : layout.generation_video_num_tokens].view(
+ batch_size, num_latent_frames, latent_hw, num_heads, head_dim
+ )
+ v_valid = v[:, : layout.generation_video_num_tokens].view(
+ batch_size, num_latent_frames, latent_hw, num_heads, head_dim
+ )
+ max_tokens_per_frame = layout.max_tokens_per_frame
+ q_padded[:, : num_latent_frames * max_tokens_per_frame].view(
+ batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
+ )[:, :, :latent_hw] = q_valid
+ k_padded[:, : num_latent_frames * max_tokens_per_frame].view(
+ batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
+ )[:, :, :latent_hw] = k_valid
+ v_padded[:, : num_latent_frames * max_tokens_per_frame].view(
+ batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
+ )[:, :, :latent_hw] = v_valid
+
+ reference_video_f, reference_video_tokens_per_frame = (
+ layout.reference_video_f,
+ layout.reference_video_tokens_per_frame,
+ )
+ reference_video_k_valid = reference_video_k[
+ :, : layout.reference_video_num_tokens
+ ].view(
+ batch_size,
+ reference_video_f,
+ reference_video_tokens_per_frame,
+ num_heads,
+ head_dim,
+ )
+ reference_video_v_valid = reference_video_v[
+ :, : layout.reference_video_num_tokens
+ ].view(
+ batch_size,
+ reference_video_f,
+ reference_video_tokens_per_frame,
+ num_heads,
+ head_dim,
+ )
+ k_padded[
+ :,
+ padded_generation_video_len : padded_generation_video_len
+ + reference_video_f * max_tokens_per_frame,
+ ].view(
+ batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim
+ )[:, :, :reference_video_tokens_per_frame] = reference_video_k_valid
+ v_padded[
+ :,
+ padded_generation_video_len : padded_generation_video_len
+ + reference_video_f * max_tokens_per_frame,
+ ].view(
+ batch_size, reference_video_f, max_tokens_per_frame, num_heads, head_dim
+ )[:, :, :reference_video_tokens_per_frame] = reference_video_v_valid
+
+ # At 0.0 the bias is an exact identity, so the kernel is not asked to apply it;
+ # otherwise the cached partial keeps the identity compiled flex guards on (_make_score_mod).
+ log_scale = float(self.log_scale)
+ _score_mod = (
+ None
+ if log_scale == 0.0
+ else _make_score_mod(max_tokens_per_frame, log_scale)
+ )
+
+ # flex_attention takes [B, H, L, D]; the default 1/sqrt(D) scale matches USPAttention.
+ attn_output_padded = _flex_attention_compiled(
+ q_padded.transpose(1, 2),
+ k_padded.transpose(1, 2),
+ v_padded.transpose(1, 2),
+ block_mask=block_mask,
+ score_mod=_score_mod,
+ ).transpose(1, 2) # -> [B, padded_generation_video_len, N, C]
+
+ attn_output_valid = attn_output_padded[
+ :, : num_latent_frames * max_tokens_per_frame
+ ].view(
+ batch_size, num_latent_frames, max_tokens_per_frame, num_heads, head_dim
+ )[:, :, :latent_hw]
+ attn_output_valid = attn_output_valid.reshape(
+ batch_size, num_latent_frames * latent_hw, num_heads, head_dim
+ )
+ if q_padding.shape[1] == 0:
+ return attn_output_valid
+ return torch.cat([attn_output_valid, q_padding], dim=1) # [B, seq_len, N, C]
+
+
+class _InContextLayout:
+ """Token geometry of the padded [generation | reference-video] in-context layout.
+
+ Three sets of sizes: this clip's token counts; those of a full-length clip, which every
+ clip is padded up to; and the full-length sizes rounded up to 128 for flex_attention.
+ The attention pads to the last set and the block mask is built from the same numbers,
+ so one compiled mask serves every clip of a request."""
+
+ def __init__(
+ self,
+ generation_video_grid: tuple[int, int, int],
+ reference_video_grid: tuple[int, int, int],
+ full_clip_grid: tuple[int, int, int],
+ ) -> None:
+ f, h, w = generation_video_grid
+ reference_video_f, reference_video_h, reference_video_w = reference_video_grid
+ full_clip_f, full_clip_h, full_clip_w = full_clip_grid
+ self.generation_video_f, self.generation_video_h, self.generation_video_w = (
+ f,
+ h,
+ w,
+ )
+ self.reference_video_f, self.reference_video_h, self.reference_video_w = (
+ reference_video_f,
+ reference_video_h,
+ reference_video_w,
+ )
+
+ # This clip; shorter than full length only for the last clip of a video.
+ self.generation_video_tokens_per_frame = h * w
+ self.reference_video_tokens_per_frame = reference_video_h * reference_video_w
+ self.generation_video_num_tokens = f * h * w
+ self.reference_video_num_tokens = (
+ reference_video_f * reference_video_h * reference_video_w
+ )
+
+ # A full-length clip. The generation video carries one extra reference-image frame,
+ # the reference video does not.
+ self.max_tokens_per_frame = full_clip_h * full_clip_w
+ self.max_generation_video_num_tokens = full_clip_f * self.max_tokens_per_frame
+ self.max_reference_video_num_tokens = (
+ full_clip_f - 1
+ ) * self.max_tokens_per_frame
+
+ # Padded to flex_attention's 128-token blocks; keys are [generation | reference video].
+ self.padded_generation_video_len = (
+ math.ceil(self.max_generation_video_num_tokens / 128) * 128
+ )
+ self.padded_reference_video_len = (
+ math.ceil(self.max_reference_video_num_tokens / 128) * 128
+ )
+ self.padded_kv_len = (
+ self.padded_generation_video_len + self.padded_reference_video_len
+ )
+
+ def block_mask_fn(self) -> Callable[..., torch.Tensor]:
+ """Returns the block mask function based on the [gen | reference-video] in-context layout.
+
+ A valid gen query attends every valid gen key plus the reference-video keys of its own frame.
+
+ keys: [generation tokens | generation pad | reference tokens | reference pad]
+ 0 padded_generation_video_len padded_kv_len
+ queries: [generation tokens | generation pad]
+ 0 padded_generation_video_len
+ """
+ tokens_per_frame = self.max_tokens_per_frame
+ num_generation_tokens = self.max_generation_video_num_tokens
+ num_reference_tokens = self.max_reference_video_num_tokens
+ reference_start = self.padded_generation_video_len
+
+ def _mask_function(
+ b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor
+ ) -> torch.Tensor:
+ # Pad positions are not tokens; pad queries attend nothing.
+ q_is_generation_token = q_idx < num_generation_tokens
+ # Generation frame of the query; frame 0 is the reference image.
+ q_frame = q_idx // tokens_per_frame
+
+ # Every generation token attends every generation token.
+ kv_is_generation_token = kv_idx < num_generation_tokens
+
+ # Reference-video tokens are attended only by the queries of their own frame.
+ kv_reference_idx = kv_idx - reference_start
+ kv_is_reference_token = (kv_idx >= reference_start) & (
+ kv_reference_idx < num_reference_tokens
+ )
+ # +1: the reference video has no counterpart for the generation stream's
+ # reference-image frame 0.
+ kv_reference_frame = kv_reference_idx // tokens_per_frame + 1
+ kv_is_same_frame_reference_token = kv_is_reference_token & (
+ q_frame == kv_reference_frame
+ )
+ return q_is_generation_token & (
+ kv_is_generation_token | kv_is_same_frame_reference_token
+ )
+
+ return _mask_function
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py
new file mode 100644
index 0000000000..ec2e4d662e
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/models/dits/wan_animate_2_clip_conditioning.py
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Per-clip contracts between the Wan-Animate-2 stages and the DiT.
+
+Lives on the model side so the DiT can type its ``clip_cond`` / ``reference_kv`` kwargs
+without importing a pipeline stage: the before-denoising stage builds the conditioning, the
+DiT's ``build_reference_kv`` turns it into the reference K/V the denoising stage hands to
+every ``forward`` of that clip.
+"""
+
+import msgspec
+import torch
+
+
+class WanAnimate2ClipConditioning(msgspec.Struct, frozen=True, kw_only=True):
+ """Per-clip conditioning consumed by the DiT's forward_ref / forward_gen."""
+
+ # Inputs for ``forward_ref`` (reference video = the motion source)
+ # VAE latents of this clip's reference-video frames,
+ # [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32.
+ reference_video_latents: torch.Tensor
+ # Generation token grid (latent_t, latent_h // 2, latent_w // 2): the DiT patchifies the
+ # latent with patch_size (1, 2, 2), so each spatial side halves.
+ generation_video_grid_sizes: tuple[int, int, int]
+ # CLIP tokens of the clip's first reference-video frame, [1, 257, 1280] bf16.
+ reference_video_frame_0_image_embeddings: torch.Tensor
+ # [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16.
+ # Channels: [0:4] given-frame mask, all ones (every reference frame is known);
+ # [4:20] VAE latent, the same tensor as reference_video_latents.
+ # Time: the clip's reference-video latent frames; no reference-image frame in front.
+ # forward_ref cats it under reference_video_latents (16 ch) for the 36-channel DiT input.
+ reference_video_condition: torch.Tensor
+ # T5 embedding of prompt_ref, [L_prompt_ref, 4096] bf16.
+ prompt_ref_embeddings: torch.Tensor
+
+ # Inputs for: ``forward_gen``
+ # [20, latent_t, latent_h, latent_w] bf16.
+ # Channels: [0:4] given-frame mask (1 = known pixels, 0 = to generate);
+ # [4:20] VAE latent of the given pixels, encoded zeros elsewhere.
+ # Time: frame 0 is the reference image (mask = all ones); frames 1..latent_t are the clip's
+ # own clip_latent_t frames, given only for the num_frames_conditioning frames fed
+ # back from the previous clip (none for clip 0).
+ # forward_gen cats it under the noise latent (16 ch) for the 36-channel DiT input.
+ generation_condition: torch.Tensor
+ # Reference-video token grid (reference_video_latent_t, _h // 2, _w // 2); same halving.
+ reference_video_grid_sizes: tuple[int, int, int]
+ # Token grid of a full-length clip (the configured clip_len); the attention layout and
+ # block mask are sized from it even on the shorter last clip, so one compiled mask
+ # serves the whole request.
+ full_clip_grid_sizes: tuple[int, int, int]
+
+ # Pixel-frame count of this clip (the last clip may be shorter than clip_len).
+ num_frames: int
+ # Seeded initial noise for the denoising loop, [16, latent_t, latent_h, latent_w] fp32.
+ init_noise: torch.Tensor
+
+
+class WanAnimate2ReferenceKV(msgspec.Struct, frozen=True):
+ """One clip's post-RoPE reference-video K and V, keyed by DiT block index.
+
+ Returned by ``WanAnimate2Transformer3DModel.build_reference_kv`` and owned by the caller
+ for that clip only; the DiT keeps no copy between calls.
+ """
+
+ # [1, S_reference_video, local_heads, head_dim] in the DiT param dtype. Under Ulysses SP
+ # this is the full sequence with sharded heads, the layout forward_gen uses after its a2a.
+ k: dict[int, torch.Tensor]
+ v: dict[int, torch.Tensor]
diff --git a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py
index 66e40dc2c8..01ea542dd7 100755
--- a/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py
+++ b/python/sglang/multimodal_gen/runtime/models/dits/wanvideo.py
@@ -617,6 +617,61 @@ class WanTransformerBlock(nn.Module):
self.scale_shift_table = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
+ def _qkv(
+ self, norm_hidden_states: torch.Tensor, local_heads: int
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ query, _ = self.to_q(norm_hidden_states)
+ key, _ = self.to_k(norm_hidden_states)
+ value, _ = self.to_v(norm_hidden_states)
+ if self.norm_q is not None:
+ query = (
+ tensor_parallel_rms_norm(query, self.norm_q)
+ if self.tp_rmsnorm
+ else self.norm_q(query)
+ )
+ if self.norm_k is not None:
+ key = (
+ tensor_parallel_rms_norm(key, self.norm_k)
+ if self.tp_rmsnorm
+ else self.norm_k(key)
+ )
+ return tuple(
+ tensor.squeeze(1).unflatten(2, (local_heads, self.dim_head))
+ for tensor in (query, key, value)
+ )
+
+ def _cross_and_ffn(
+ self,
+ hidden_states: torch.Tensor,
+ attn_output: torch.Tensor,
+ encoder_hidden_states: torch.Tensor,
+ gate_msa: torch.Tensor,
+ c_shift_msa: torch.Tensor,
+ c_scale_msa: torch.Tensor,
+ c_gate_msa: torch.Tensor,
+ orig_dtype: torch.dtype,
+ ) -> torch.Tensor:
+ attn_output, _ = self.to_out(attn_output)
+ attn_output = attn_output.squeeze(1)
+ null_shift = null_scale = torch.zeros(
+ (1,), device=hidden_states.device, dtype=hidden_states.dtype
+ )
+ norm_hidden_states, hidden_states = self.self_attn_residual_norm(
+ hidden_states, attn_output, gate_msa, null_shift, null_scale
+ )
+ norm_hidden_states = norm_hidden_states.to(orig_dtype)
+ hidden_states = hidden_states.to(orig_dtype)
+ attn_output = self.attn2(
+ norm_hidden_states, context=encoder_hidden_states, context_lens=None
+ )
+ norm_hidden_states, hidden_states = self.cross_attn_residual_norm(
+ hidden_states, attn_output, 1, c_shift_msa, c_scale_msa
+ )
+ norm_hidden_states = norm_hidden_states.to(orig_dtype)
+ hidden_states = hidden_states.to(orig_dtype)
+ ff_output = self.ffn(norm_hidden_states)
+ return self.mlp_residual(ff_output, c_gate_msa, hidden_states).to(orig_dtype)
+
def forward(
self,
hidden_states: torch.Tensor,
@@ -655,23 +710,7 @@ class WanTransformerBlock(nn.Module):
# 1. Self-attention
norm_hidden_states = self.norm1(hidden_states, shift_msa, scale_msa)
- query, _ = self.to_q(norm_hidden_states)
- key, _ = self.to_k(norm_hidden_states)
- value, _ = self.to_v(norm_hidden_states)
-
- if self.norm_q is not None:
- if self.tp_rmsnorm:
- query = tensor_parallel_rms_norm(query, self.norm_q)
- else:
- query = self.norm_q(query)
- if self.norm_k is not None:
- if self.tp_rmsnorm:
- key = tensor_parallel_rms_norm(key, self.norm_k)
- else:
- key = self.norm_k(key)
- query = query.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
- key = key.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
- value = value.squeeze(1).unflatten(2, (self.local_num_heads, self.dim_head))
+ query, key, value = self._qkv(norm_hidden_states, self.local_num_heads)
# Apply rotary embeddings
cos, sin = freqs_cis
@@ -729,38 +768,16 @@ class WanTransformerBlock(nn.Module):
attn_output = self.attn1(query, key, value)
attn_output = attn_output.flatten(2)
- attn_output, _ = self.to_out(attn_output)
- attn_output = attn_output.squeeze(1)
-
- null_shift = null_scale = torch.zeros(
- (1,), device=hidden_states.device, dtype=hidden_states.dtype
+ return self._cross_and_ffn(
+ hidden_states,
+ attn_output,
+ encoder_hidden_states,
+ gate_msa,
+ c_shift_msa,
+ c_scale_msa,
+ c_gate_msa,
+ orig_dtype,
)
- norm_hidden_states, hidden_states = self.self_attn_residual_norm(
- hidden_states, attn_output, gate_msa, null_shift, null_scale
- )
- norm_hidden_states, hidden_states = (
- norm_hidden_states.to(orig_dtype),
- hidden_states.to(orig_dtype),
- )
-
- # 2. Cross-attention
- attn_output = self.attn2(
- norm_hidden_states, context=encoder_hidden_states, context_lens=None
- )
- norm_hidden_states, hidden_states = self.cross_attn_residual_norm(
- hidden_states, attn_output, 1, c_shift_msa, c_scale_msa
- )
- norm_hidden_states, hidden_states = (
- norm_hidden_states.to(orig_dtype),
- hidden_states.to(orig_dtype),
- )
-
- # 3. Feed-forward
- ff_output = self.ffn(norm_hidden_states)
- hidden_states = self.mlp_residual(ff_output, c_gate_msa, hidden_states)
- hidden_states = hidden_states.to(orig_dtype)
-
- return hidden_states
class WanTransformerBlock_VSA(nn.Module):
@@ -1057,29 +1074,9 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
)
# 3. Transformer blocks
- attn_backend = get_global_server_args().attention_backend
- transformer_block = (
- WanTransformerBlock_VSA
- if (attn_backend and attn_backend.lower() == "video_sparse_attn")
- else WanTransformerBlock
- )
self.blocks = nn.ModuleList(
[
- transformer_block(
- inner_dim,
- config.ffn_dim,
- config.num_attention_heads,
- config.qk_norm,
- config.cross_attn_norm,
- config.eps,
- config.added_kv_proj_dim,
- self._supported_attention_backends
- | {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
- prefix=f"blocks.{i}",
- attention_type=config.attention_type,
- sla_topk=config.sla_topk,
- quant_config=quant_config,
- )
+ self._make_block(index=i, config=config, quant_config=quant_config)
for i in range(config.num_layers)
]
)
@@ -1127,6 +1124,36 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.layer_names = ["blocks"]
+ def _make_block(
+ self,
+ *,
+ index: int,
+ config: WanVideoConfig,
+ quant_config: QuantizationConfig | None,
+ ) -> nn.Module:
+ """Block factory; subclasses override it to substitute their block class."""
+ attn_backend = get_global_server_args().attention_backend
+ transformer_block = (
+ WanTransformerBlock_VSA
+ if (attn_backend and attn_backend.lower() == "video_sparse_attn")
+ else WanTransformerBlock
+ )
+ return transformer_block(
+ config.num_attention_heads * config.attention_head_dim,
+ config.ffn_dim,
+ config.num_attention_heads,
+ config.qk_norm,
+ config.cross_attn_norm,
+ config.eps,
+ config.added_kv_proj_dim,
+ self._supported_attention_backends
+ | {AttentionBackendEnum.VIDEO_SPARSE_ATTN},
+ prefix=f"blocks.{index}",
+ attention_type=config.attention_type,
+ sla_topk=config.sla_topk,
+ quant_config=quant_config,
+ )
+
@lru_cache(maxsize=1)
def _compute_rope_for_sequence_shard(
self,
@@ -1338,30 +1365,37 @@ class WanTransformer3DModel(CachableDiT, LayerwiseOffloadableModuleMixin):
if seq_shard_pad > 0:
hidden_states = hidden_states[:, :seq_len_orig, :]
- # 5. Output norm, projection & unpatchify
- if temb.dim() == 3:
+ return self._unpatchify(
+ hidden_states,
+ temb,
+ (post_patch_num_frames, post_patch_height, post_patch_width),
+ )
+
+ def _unpatchify(
+ self,
+ hidden_states: torch.Tensor,
+ timestep_embeddings: torch.Tensor,
+ grid_size: tuple[int, int, int],
+ ) -> torch.Tensor:
+ """Output norm, projection and token-grid reconstruction for Wan variants."""
+ if timestep_embeddings.dim() == 3:
# batch_size, seq_len, inner_dim (wan 2.2 ti2v)
shift, scale = (
- self.scale_shift_table.unsqueeze(0) + temb.unsqueeze(2)
+ self.scale_shift_table.unsqueeze(0) + timestep_embeddings.unsqueeze(2)
).chunk(2, dim=2)
shift = shift.squeeze(2)
scale = scale.squeeze(2)
else:
# batch_size, inner_dim
- shift, scale = (self.scale_shift_table + temb.unsqueeze(1)).chunk(2, dim=1)
+ shift, scale = (
+ self.scale_shift_table + timestep_embeddings.unsqueeze(1)
+ ).chunk(2, dim=1)
hidden_states = self.norm_out(hidden_states, shift, scale)
hidden_states, _ = self.proj_out(hidden_states)
hidden_states = hidden_states.reshape(
- batch_size,
- post_patch_num_frames,
- post_patch_height,
- post_patch_width,
- p_t,
- p_h,
- p_w,
- -1,
+ hidden_states.shape[0], *grid_size, *self.patch_size, -1
)
hidden_states = hidden_states.permute(0, 7, 1, 4, 2, 5, 3, 6)
output = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3)
diff --git a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py
index 03c695909b..a6340c7d07 100644
--- a/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py
+++ b/python/sglang/multimodal_gen/runtime/models/schedulers/scheduling_dpm_solver_multistep.py
@@ -9,6 +9,7 @@
# This wrapper delegates all numerical work to diffusers' implementation
# and only adapts the interface for sglang's denoising stage.
+import numpy as np
import torch
from diffusers import (
DPMSolverMultistepScheduler as DiffusersDPMSolverMultistepScheduler,
@@ -101,10 +102,43 @@ class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin, BaseScheduler):
def begin_index(self) -> int | None:
return self._begin_index
- def set_timesteps(self, num_inference_steps: int, device=None, **kwargs):
- self._inner.set_timesteps(num_inference_steps, device=device, **kwargs)
+ def set_timesteps(
+ self,
+ num_inference_steps: int | None = None,
+ device=None,
+ sigmas: np.ndarray | list[float] | None = None,
+ **kwargs,
+ ):
+ if sigmas is None:
+ self._inner.set_timesteps(num_inference_steps, device=device, **kwargs)
+ else:
+ self._set_sigmas(np.asarray(sigmas, dtype=np.float64), device)
self.timesteps = self._inner.timesteps
+ def _set_sigmas(self, sigmas: np.ndarray, device=None) -> None:
+ inner = self._inner
+ if inner.config.final_sigmas_type == "sigma_min":
+ sigma_last = (
+ (1 - inner.alphas_cumprod[0]) / inner.alphas_cumprod[0]
+ ) ** 0.5
+ else:
+ sigma_last = 0
+ timesteps = sigmas * inner.config.num_train_timesteps
+ sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32)
+ inner.sigmas = torch.from_numpy(sigmas)
+ inner.timesteps = torch.from_numpy(timesteps).to(
+ device=device, dtype=torch.int64
+ )
+ inner.num_inference_steps = len(timesteps)
+ self._reset_solver_state()
+
+ def _reset_solver_state(self) -> None:
+ inner = self._inner
+ inner.model_outputs = [None] * inner.config.solver_order
+ inner.lower_order_nums = 0
+ inner._step_index = None
+ inner._begin_index = None
+
def scale_model_input(
self, sample: torch.Tensor, timestep: int | None = None
) -> torch.Tensor:
diff --git a/python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py
new file mode 100644
index 0000000000..e2df14e238
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines/wan_animate_2_pipeline.py
@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Wan-Animate-2 pipeline.
+
+Stages: WanAnimate2BeforeDenoisingStage (per-clip conditioning) -> WanAnimate2DenoisingStage
+(per-clip denoise + in-loop VAE decode) -> WanAnimate2OutputStage (emit frames).
+Routed by model_index.json ``_class_name == "WanAnimate2Pipeline"``; config comes from
+the registry detector (Wan_Animate_2_14B_Config). Every component loads through the
+standard component loaders. Text uses TextEncodingStage; CLIP and VAE adapters
+preserve the reference preprocessing and latent scaling.
+"""
+
+from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
+from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
+from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
+ TransformerLoader,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
+ ComposedPipelineBase,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2 import (
+ WanAnimate2BeforeDenoisingStage,
+ WanAnimate2DenoisingStage,
+ WanAnimate2ImageEncoderAdapter,
+ WanAnimate2OutputStage,
+ WanAnimate2VaeAdapter,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+
+class WanAnimate2TransformerLoader(TransformerLoader):
+ """The checkpoint is the plain Wan2.2-I2V-14B backbone, so every key must map."""
+
+ component_names = []
+ strict_checkpoint_keys = True
+
+
+class WanAnimate2Pipeline(ComposedPipelineBase):
+ """Wan-Animate-2 14B in-context reference image + reference video -> video pipeline."""
+
+ pipeline_name = "WanAnimate2Pipeline"
+ is_video_pipeline = True
+
+ # All seven model_index.json components. The scheduler is built from the
+ # checkpoint's scheduler/scheduler_config.json (DPM-Solver++, flow prediction,
+ # flow sigmas) by the shared loader. Two of its settings are overridden at
+ # runtime: the stages call set_timesteps(sigmas=...) with the official Wan
+ # sigma grid at every clip (sigma_0 is exactly 1.0), shifted by
+ # pipeline_config.flow_shift, so the JSON's flow_shift and timestep_spacing
+ # are not consulted. The JSON's beta_schedule only shapes the beta-derived
+ # tables, which the flow-sigma path does not read.
+ _required_config_modules: list[str] = [
+ "text_encoder",
+ "tokenizer",
+ "vae",
+ "transformer",
+ "image_encoder",
+ "image_processor",
+ "scheduler",
+ ]
+ component_loaders = {"transformer": WanAnimate2TransformerLoader}
+
+ def create_pipeline_stages(self, server_args: ServerArgs) -> None:
+ # Keep native modules visible to residency and layerwise-offload management;
+ # adapters only translate the model-specific stage API.
+ device = get_local_torch_device()
+ image_encoder = WanAnimate2ImageEncoderAdapter.from_loaded(
+ self.get_module("image_encoder"),
+ image_processor=self.get_module("image_processor"),
+ device=device,
+ )
+ vae = WanAnimate2VaeAdapter(self.get_module("vae"))
+ self.add_stage(
+ stage_name="wan_animate_2_before_denoising",
+ stage=WanAnimate2BeforeDenoisingStage(
+ vae=vae,
+ image_encoder=image_encoder,
+ text_encoder=self.get_module("text_encoder"),
+ tokenizer=self.get_module("tokenizer"),
+ pipeline_config=server_args.pipeline_config,
+ scheduler=self.get_module("scheduler"),
+ ),
+ )
+ # Decode each clip before constructing the following clip's conditions.
+ self.add_stage(
+ stage_name="wan_animate_2_denoising",
+ stage=WanAnimate2DenoisingStage(
+ transformer=self.get_module("transformer"),
+ scheduler=self.get_module("scheduler"),
+ pipeline=self,
+ vae=vae,
+ image_encoder=image_encoder,
+ ),
+ )
+ # Replaces the standard DecodingStage: frames are already decoded in the loop.
+ self.add_stage_factory(
+ RoleType.DECODER,
+ lambda: WanAnimate2OutputStage(),
+ "decoding_stage",
+ )
+
+
+EntryClass = WanAnimate2Pipeline
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py
index 8e9cc4987a..f8ffc1b0bb 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py
@@ -234,8 +234,31 @@ class DecodingStage(PipelineStage):
Returns:
Decoded video tensor with shape (batch, channels, frames, height, width),
- normalized to [0, 1] range and moved to CPU as float32
+ normalized to [0, 1] on the decode device.
"""
+ normalized_on_frames = (
+ (lambda frames: on_frames(_to_unit_range(frames)))
+ if on_frames is not None
+ else None
+ )
+ image = self.decode_raw(
+ latents,
+ server_args,
+ vae_dtype=vae_dtype,
+ on_frames=normalized_on_frames,
+ )
+ return _to_unit_range(image)
+
+ @torch.no_grad()
+ def decode_raw(
+ self,
+ latents: torch.Tensor,
+ server_args: ServerArgs,
+ *,
+ vae_dtype: torch.dtype,
+ on_frames=None,
+ ) -> torch.Tensor:
+ """Decode without pixel normalization, including for inter-clip conditioning."""
latents = latents.to(get_local_torch_device())
# The caller resolves the decode-only override before component use so
# residency and execution agree on the target dtype.
@@ -270,9 +293,7 @@ class DecodingStage(PipelineStage):
latents = latents.to(vae_dtype)
decode_kwargs = {}
if on_frames is not None:
- decode_kwargs["on_frames"] = lambda frames: on_frames(
- _to_unit_range(frames)
- )
+ decode_kwargs["on_frames"] = on_frames
with temporary_module_dtype(
self.vae, vae_dtype, enabled=should_cast_vae
) as vae:
@@ -304,8 +325,6 @@ class DecodingStage(PipelineStage):
raise
image = _ensure_tensor_decode_output(decode_output)
- # De-normalize image to [0, 1] range
- image = _to_unit_range(image)
return image
def load_model(self):
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py
index 7de0fadc7d..7012258b9f 100644
--- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/denoising.py
@@ -2084,11 +2084,40 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
neg_cond_kwargs=ctx.neg_cond_kwargs,
guidance=ctx.guidance,
)
+ timesteps_cpu = self._run_denoising_loop(ctx, batch, server_args)
+
+ # collect rollout outputs before finalization gathers or replaces latents
+ if batch.rollout:
+ self._postprocess_rollout_outputs(
+ batch=batch,
+ latents=ctx.latents,
+ num_inference_steps=len(timesteps_cpu),
+ final_timestep=timesteps_cpu.new_zeros(()),
+ server_args=server_args,
+ )
+ self._finalize_denoising_loop(ctx, batch, server_args)
+ return batch
+
+ def _run_denoising_loop(
+ self,
+ ctx: DenoisingContext,
+ batch: Req,
+ server_args: ServerArgs,
+ *,
+ collect_trajectory: bool = True,
+ ) -> torch.Tensor:
+ """Run a prepared loop, including profiling and DiT residency.
+
+ Clip-based models can reuse this without finalizing the entire request.
+ """
denoising_start_time = time.time()
self._before_denoising_loop(ctx, batch, server_args)
# to avoid device-sync caused by timestep comparison
timesteps_cpu = ctx.timesteps.cpu()
num_timesteps = timesteps_cpu.shape[0]
+ progress_step_interval = ctx.extra.get(
+ "progress_step_interval", ctx.scheduler.order
+ )
# Re-resolve the explicit-range gate so the per-step markers
# below honor this request's is_warmup state. Layer hooks are
# registered by the residency manager at the use-site.
@@ -2132,7 +2161,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
# BEFORE _run_denoising_step so ctx.latents is still the
# pre-step value. Gated on batch.rollout to keep the
# non-rollout path strictly untouched.
- if batch.rollout:
+ if collect_trajectory and batch.rollout:
batch._rollout_loop_step_index = step_index
self._maybe_append_dit_trajectory_step(
batch=batch,
@@ -2141,11 +2170,12 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
step_index=step_index,
)
self._run_denoising_step(ctx, step, batch, server_args)
- self._record_trajectory(ctx, step, batch, server_args)
+ if collect_trajectory:
+ self._record_trajectory(ctx, step, batch, server_args)
if step_index == num_timesteps - 1 or (
(step_index + 1) > ctx.num_warmup_steps
- and (step_index + 1) % ctx.scheduler.order == 0
+ and (step_index + 1) % progress_step_interval == 0
and progress_bar is not None
):
progress_bar.update()
@@ -2165,20 +2195,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
del step
self._finish_active_component_use()
- # Rollout postprocessing must run BEFORE _finalize_denoising_loop so
- # the final scheduler.step output (ctx.latents) is still SP-sharded and
- # can be gathered uniformly alongside the per-step dit_trajectory via
- # gather_stacked_latents_for_sp.
- if batch.rollout:
- self._postprocess_rollout_outputs(
- batch=batch,
- latents=ctx.latents,
- num_inference_steps=num_timesteps,
- final_timestep=timesteps_cpu.new_zeros(()),
- server_args=server_args,
- )
- self._finalize_denoising_loop(ctx, batch, server_args)
- return batch
+ return timesteps_cpu
def _get_extra_func_kwarg_names(self, func) -> tuple[bool, frozenset[str]]:
import functools
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py
new file mode 100644
index 0000000000..972b2b898b
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/__init__.py
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Wan-Animate-2-specific pipeline stages."""
+
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
+ WanAnimate2BeforeDenoisingStage,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
+ WanAnimate2DenoisingStage,
+ WanAnimate2OutputStage,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
+ WanAnimate2ImageEncoderAdapter,
+ WanAnimate2VaeAdapter,
+)
+
+__all__ = [
+ "WanAnimate2BeforeDenoisingStage",
+ "WanAnimate2DenoisingStage",
+ "WanAnimate2OutputStage",
+ "WanAnimate2ImageEncoderAdapter",
+ "WanAnimate2VaeAdapter",
+]
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py
new file mode 100644
index 0000000000..5185833642
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/audio.py
@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Reference-video audio extraction for the Wan-Animate-2 before-denoising stage."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import torch
+
+from sglang.srt.multimodal.audio_from_video import decode_audio_container
+
+# Logger callables with the ``PipelineStage.log_*`` signature: ``(msg, *args)``.
+LogFn = Callable[..., None]
+
+
+def extract_reference_video_audio(
+ reference_video_path: str,
+ *,
+ log_info: LogFn,
+ log_warning: LogFn,
+ log_no_audio_track: LogFn | None = None,
+) -> tuple[torch.Tensor | None, int | None]:
+ """Return ``(audio, sample_rate)`` for ``OutputBatch.audio``: audio ``[1, C, L]`` fp32 in
+ [-1, 1] (dim 0 is the per-output axis the save path indexes) at the track's native sample
+ rate and channel count, or ``(None, None)`` when the video has no decodable audio track.
+ The track is decoded in-process by the shared ``decode_audio_container`` reader (PyAV).
+ Best-effort: every failure is logged, none is raised. A video without an audio track goes
+ to ``log_no_audio_track`` (``log_warning`` when None); a track that cannot be decoded goes
+ to ``log_warning`` with the reader's message."""
+ try:
+ sample_rate = _audio_track_sample_rate(reference_video_path)
+ if sample_rate is None:
+ (log_no_audio_track or log_warning)(
+ "Wan-Animate-2: reference video %s has no audio track; the output is silent.",
+ reference_video_path,
+ )
+ return None, None
+ # [L, C] fp32 in [-1, 1]; the reader needs a target rate, so the native one keeps the
+ # track unresampled.
+ samples = decode_audio_container(
+ reference_video_path, target_sr=sample_rate, mono=False
+ )
+ except Exception as e: # audio is best-effort: a failure never fails the request
+ log_warning(
+ "Wan-Animate-2: cannot decode the audio track of reference video %s (%s); "
+ "the output is silent.",
+ reference_video_path,
+ e,
+ )
+ return None, None
+
+ audio = torch.from_numpy(samples).T.contiguous().unsqueeze(0) # [1, C, L]
+ log_info(
+ "Wan-Animate-2: extracted reference-video audio %s @ %d Hz.",
+ tuple(audio.shape),
+ sample_rate,
+ )
+ return audio, sample_rate
+
+
+def _audio_track_sample_rate(path: str) -> int | None:
+ """Native sample rate of the video's first audio stream; None when it has none."""
+ import av
+
+ with av.open(path) as container:
+ if not container.streams.audio:
+ return None
+ return int(container.streams.audio[0].rate)
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py
new file mode 100644
index 0000000000..b36b00cbb6
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/before_denoising.py
@@ -0,0 +1,706 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Wan-Animate-2 before-denoising stage.
+
+Builds the per-clip conditioning container (``WanAnimate2ClipConditioning``) the DiT
+reads via its ``clip_cond`` kwarg, plus the clip-invariant text/CLIP/VAE encodings.
+Preprocessing helpers live in ``preprocess.py`` next to this module.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+import cv2
+import msgspec
+import numpy as np
+import torch
+from einops import rearrange
+
+from sglang.multimodal_gen.runtime.distributed import (
+ get_local_torch_device,
+ get_sp_world_size,
+)
+from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
+ ComponentUse,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
+ WanAnimate2ClipConditioning,
+)
+from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
+ DPMSolverMultistepScheduler,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio import (
+ extract_reference_video_audio,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
+ LetterboxInfo,
+ get_padding_len,
+ make_conditioning_mask,
+ read_reference_video_frames,
+ resize_by_area,
+ validate_and_get_single_string,
+ zigzag_padding,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
+ TextEncodingStage,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+
+if TYPE_CHECKING:
+ from sglang.multimodal_gen.configs.pipeline_configs.wan import (
+ Wan_Animate_2_14B_Config,
+ )
+ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
+ WanAnimate2ImageEncoderAdapter,
+ WanAnimate2VaeAdapter,
+ )
+
+# Frames of the previous clip's decoded output that open the next clip as given (not denoised);
+# the official pipelines/wan_animate_2_pipeline.py hard-codes num_frames_conditioning = 1.
+_NUM_FRAMES_CONDITIONING = 1
+# Wan VAE temporal compression: 4 pixel frames per latent frame after the first.
+_VAE_TEMPORAL_COMPRESSION = 4
+# Wan VAE latent channels.
+_NUM_LATENT_CHANNELS = 16
+
+
+def generation_token_grid(
+ num_frames: int, latent_h: int, latent_w: int
+) -> tuple[int, int, int]:
+ """DiT token grid (f, h, w) of a clip of ``num_frames`` pixel frames. Wan VAE: 4k + 1 pixel
+ frames -> k + 1 latent frames; one latent frame is prepended for the reference image;
+ the DiT patchifies with patch_size (1, 2, 2), so each spatial side halves."""
+ clip_latent_t = (num_frames - 1) // _VAE_TEMPORAL_COMPRESSION + 1
+ return clip_latent_t + 1, latent_h // 2, latent_w // 2
+
+
+def _generation_latent_shape(
+ latent_h: int, latent_w: int, num_frames: int
+) -> tuple[int, int, int, int]:
+ """``[16, latent_t, latent_h, latent_w]`` of the latent the DiT denoises for a clip of
+ ``num_frames`` pixel frames; latent_t comes from generation_token_grid (clip frames plus
+ the leading reference-image slot)."""
+ latent_t = generation_token_grid(num_frames, latent_h, latent_w)[0]
+ return (_NUM_LATENT_CHANNELS, latent_t, latent_h, latent_w)
+
+
+def get_sampling_sigmas(sampling_steps: int, shift: float) -> np.ndarray:
+ """Official Wan sampler grid: ``[sampling_steps]`` fp64 sigmas, uniform on [1, 0) with ``shift`` applied (sigma_0 == 1)."""
+ sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps]
+ sigma = shift * sigma / (1 + (shift - 1) * sigma)
+ return sigma
+
+
+class _PerClipDenoisingMetadata(msgspec.Struct, frozen=True, kw_only=True):
+ """Schedule metadata for one clip of the reference video."""
+
+ # Position in the schedule, 0-based.
+ clip_index: int
+ # Start frame (inclusive) in the padded reference video.
+ frame_start_index: int
+ # End frame (exclusive) in the padded reference video.
+ frame_end_index: int
+ # Pixel-frame count, frame_end_index - frame_start_index, zigzag-padded frames included;
+ # shorter than clip_len only on the last clip.
+ num_frames: int
+ # Leading frames taken from the previous clip's output: 0 for clip 0, else
+ # num_frames_conditioning.
+ num_frames_conditioning_for_clip: int
+
+
+def build_schedule(
+ reference_video_frames: list[np.ndarray],
+ clip_len: int,
+ num_frames_conditioning: int,
+) -> list[_PerClipDenoisingMetadata]:
+ """Prepare denoising metadata for all the clips of the reference video."""
+ schedule: list[_PerClipDenoisingMetadata] = []
+ start = 0
+ clip_index = 0
+ num_frames_reference_video = len(reference_video_frames)
+ while True:
+ if start + num_frames_conditioning >= num_frames_reference_video:
+ break
+ num_frames_conditioning_for_clip = 0 if start == 0 else num_frames_conditioning
+ clip_len_current = clip_len
+ if num_frames_reference_video - start < clip_len: # last clip shrinks
+ clip_len_current = num_frames_reference_video - start
+ schedule.append(
+ _PerClipDenoisingMetadata(
+ clip_index=clip_index,
+ frame_start_index=start,
+ frame_end_index=start + clip_len_current,
+ num_frames=clip_len_current,
+ num_frames_conditioning_for_clip=num_frames_conditioning_for_clip,
+ )
+ )
+ start += clip_len_current - num_frames_conditioning
+ clip_index += 1
+ if not schedule:
+ raise ValueError(
+ "Wan-Animate-2 reference video produced 0 clips; check "
+ "clip_len/num_frames_conditioning and that the video decoded frames."
+ )
+ return schedule
+
+
+class _WanAnimate2Inputs(msgspec.Struct, frozen=True, kw_only=True):
+ """The per-request inputs read off the Req."""
+
+ reference_image_path: str
+ reference_video_path: str
+
+ prompt: str
+ prompt_ref: str
+ negative_prompt: str
+
+ width: int
+ height: int
+ clip_len: int
+ num_frames_conditioning: int
+ fps: int
+ seed: int
+ num_inference_steps: int
+ guidance_scale: float
+ enable_audio: bool
+
+ @classmethod
+ def from_request(cls, batch: Req) -> _WanAnimate2Inputs:
+ """Read the Wan-Animate-2 input fields off the Req (all SamplingParam-backed)."""
+ reference_image_path = validate_and_get_single_string(
+ batch.image_path, "image_path"
+ )
+ reference_video_path = validate_and_get_single_string(
+ batch.video_path, "video_path"
+ )
+ prompt = validate_and_get_single_string(batch.prompt, "prompt")
+ prompt_ref = validate_and_get_single_string(batch.prompt_ref, "prompt_ref")
+ # negative_prompt_embeddings is always computed; an unset negative prompt encodes "".
+ negative_prompt = validate_and_get_single_string(
+ batch.negative_prompt or "", "negative_prompt"
+ )
+
+ seed = batch.seed
+ if isinstance(seed, list) and len(seed) != 1:
+ raise ValueError(
+ f"Wan-Animate-2 takes exactly one 'seed' per request, got {len(seed)}"
+ )
+ if isinstance(seed, list):
+ seed = seed[0]
+
+ return _WanAnimate2Inputs(
+ reference_image_path=reference_image_path,
+ reference_video_path=reference_video_path,
+ prompt=prompt,
+ prompt_ref=prompt_ref,
+ negative_prompt=negative_prompt,
+ width=int(batch.width),
+ height=int(batch.height),
+ clip_len=int(batch.clip_len),
+ num_frames_conditioning=_NUM_FRAMES_CONDITIONING,
+ fps=int(batch.fps),
+ seed=int(seed),
+ num_inference_steps=int(batch.num_inference_steps),
+ guidance_scale=float(batch.guidance_scale),
+ enable_audio=bool(batch.enable_audio),
+ )
+
+
+class WanAnimate2RequestState(msgspec.Struct, kw_only=True):
+ """Clip-independent state, computed once, reused by every clip."""
+
+ device: torch.device
+ sp_size: int
+ generator: torch.Generator
+ inputs: _WanAnimate2Inputs
+
+ # reference image
+ letterbox_info: LetterboxInfo
+ reference_image_height: int
+ reference_image_width: int
+ latent_h: int # reference_image_height // 8
+ latent_w: int # reference_image_width // 8
+ reference_image_condition: torch.Tensor # [20, 1, latent_h, latent_w] bf16
+ reference_image_embeddings: torch.Tensor # [1, 257, 1280] bf16
+
+ # text
+ prompt_embeddings: torch.Tensor # [L, 4096] bf16
+ negative_prompt_embeddings: torch.Tensor # [L, 4096] bf16
+ prompt_ref_embeddings: torch.Tensor # [L_prompt_ref, 4096] bf16
+
+ # reference video, zigzag-padded to the clip schedule
+ reference_video_frames: list[np.ndarray] # resized [H, W, 3] uint8 RGB frames
+ schedule: list[_PerClipDenoisingMetadata]
+ # frame count before padding; the final video is trimmed to it
+ num_reference_video_frames: int
+
+ # Token grid of a full-length clip; every clip's attention layout is padded to it.
+ full_clip_grid_sizes: tuple[int, int, int]
+
+ # Reference-video audio, opt-in per request via enable_audio; None gives a silent mp4.
+ audio: torch.Tensor | None = None # [1, C, L] fp32 in [-1, 1]
+ audio_sample_rate: int | None = None
+ # Assembled by the denoising stage for the output stage, [T, H, W, C] uint8.
+ decoded_frames: np.ndarray | None = None
+ # Initial noise for clip 0, drawn by the before-denoising stage so that batch.latents is
+ # the real denoising input; later clips draw from the generator in schedule order.
+ clip_0_init_noise: torch.Tensor | None = None
+
+
+# The one batch.extra entry Wan-Animate-2 uses; later stages read it back through
+# request_state_from_batch.
+WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY = "wan_animate_2_request_state"
+
+
+def request_state_from_batch(batch: Req) -> WanAnimate2RequestState:
+ state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
+ if not isinstance(state, WanAnimate2RequestState):
+ raise ValueError(
+ f"batch.extra[{WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY!r}] is missing: "
+ "WanAnimate2BeforeDenoisingStage must run before this stage."
+ )
+ return state
+
+
+def _mirror_request_state_into_batch(
+ batch: Req,
+ request_state: WanAnimate2RequestState,
+ *,
+ scheduler: DPMSolverMultistepScheduler,
+ flow_shift: float,
+) -> None:
+ """Populate the standard DenoisingStage input fields from the request state.
+
+ The clip loop keeps reading ``request_state``; the embeddings, generator, sigma grid and
+ timesteps exist for the shared stage verification, warmup and residency bookkeeping.
+ ``batch.latents`` is clip 0's initial noise, drawn here (the seed's first draw) and
+ consumed by ``build_clip_conditioning`` for clip 0, so the sequence of generator draws is
+ the same as when the loop drew it: numerics are unchanged."""
+ inputs = request_state.inputs
+ batch.prompt_embeds = [request_state.prompt_embeddings]
+ batch.negative_prompt_embeds = [request_state.negative_prompt_embeddings]
+ batch.image_embeds = [request_state.reference_image_embeddings]
+ batch.generator = request_state.generator
+
+ sigmas = get_sampling_sigmas(inputs.num_inference_steps, flow_shift)
+ scheduler.set_timesteps(sigmas=sigmas, device=request_state.device)
+ batch.timesteps = scheduler.timesteps
+ batch.sigmas = sigmas.tolist()
+
+ batch.latents = _draw_clip_0_init_noise(request_state)
+ batch.raw_latent_shape = batch.latents.shape
+
+
+def _draw_clip_0_init_noise(request_state: WanAnimate2RequestState) -> torch.Tensor:
+ """Draw clip 0's initial noise from the request generator (its first draw) and keep it on
+ the request state for build_clip_conditioning; clips 1.. then take the following draws."""
+ init_noise = torch.randn(
+ *_generation_latent_shape(
+ request_state.latent_h,
+ request_state.latent_w,
+ request_state.schedule[0].num_frames,
+ ),
+ dtype=torch.float32,
+ device=request_state.device,
+ generator=request_state.generator,
+ )
+ request_state.clip_0_init_noise = init_noise
+ return init_noise
+
+
+VaeEncoderFn = Callable[[list[torch.Tensor]], list[torch.Tensor]]
+ImageEmbedderFn = Callable[[list[torch.Tensor]], torch.Tensor]
+
+
+def build_clip_conditioning(
+ request_state: WanAnimate2RequestState,
+ clip_denoising_metadata: _PerClipDenoisingMetadata,
+ prev_clip_conditioning_frames: torch.Tensor | None,
+ *,
+ vae_encoder: VaeEncoderFn,
+ image_embedder: ImageEmbedderFn,
+) -> WanAnimate2ClipConditioning:
+ """Build conditioning for a single clip."""
+ device = request_state.device
+ latent_h, latent_w = request_state.latent_h, request_state.latent_w
+ reference_image_height, reference_image_width = (
+ request_state.reference_image_height,
+ request_state.reference_image_width,
+ )
+
+ # gen/noise latent geometry
+ num_frames = clip_denoising_metadata.num_frames
+ grid_sizes = generation_token_grid(num_frames, latent_h, latent_w)
+ latent_t = grid_sizes[0]
+ # The clip's own latent frames, without the prepended reference-image frame.
+ clip_latent_t = latent_t - 1
+ init_noise_shape = _generation_latent_shape(latent_h, latent_w, num_frames)
+
+ with (
+ torch.autocast(device_type=device.type, dtype=torch.bfloat16),
+ torch.no_grad(),
+ ):
+ # previous-clip condition over the clip's own latent frames (no reference-image frame)
+ # [4, clip_latent_t, latent_h, latent_w] fp32
+ prev_clip_condition_mask = make_conditioning_mask(
+ clip_latent_t,
+ latent_h,
+ latent_w,
+ clip_denoising_metadata.num_frames_conditioning_for_clip,
+ device=device,
+ )
+ # The clip's pixel video: the given frames first, zeros where the DiT generates.
+ # Built on ``device`` so the frames never leave the GPU.
+ prev_clip_condition_pixel_space = torch.zeros(
+ 3,
+ num_frames,
+ reference_image_height,
+ reference_image_width,
+ device=device,
+ )
+ if clip_denoising_metadata.num_frames_conditioning_for_clip > 0:
+ # prev_clip_conditioning_frames: the previous clip's last
+ # ``num_frames_conditioning`` decoded frames,
+ # [C, num_frames_conditioning, reference_image_height, reference_image_width] bf16 in [-1, 1].
+ if prev_clip_conditioning_frames is None:
+ raise ValueError(
+ "For all clips except clip-0 `prev_clip_conditioning_frames` must be not None."
+ )
+
+ prev_clip_condition_pixel_space[
+ :, : clip_denoising_metadata.num_frames_conditioning_for_clip
+ ] = torch.nn.functional.interpolate(
+ prev_clip_conditioning_frames[
+ :, : clip_denoising_metadata.num_frames_conditioning_for_clip
+ ],
+ size=(reference_image_height, reference_image_width),
+ mode="bicubic",
+ )
+
+ # [16, clip_latent_t, latent_h, latent_w] fp32
+ prev_clip_condition = vae_encoder([prev_clip_condition_pixel_space])[0]
+
+ # [20, clip_latent_t, latent_h, latent_w] bf16
+ prev_clip_condition = torch.concat(
+ [prev_clip_condition_mask, prev_clip_condition]
+ ).to(torch.bfloat16)
+
+ # [20, latent_t, latent_h, latent_w] bf16
+ # recall: latent_t = clip_latent_t + 1 (+1 is for the reference image condition in latent space)
+ generation_condition = torch.concat(
+ [request_state.reference_image_condition, prev_clip_condition], dim=1
+ )
+
+ # reference-video slice -> reference_video_latents
+ # [t, h, w, c] uint8
+ reference_video_clip_np = np.stack(
+ request_state.reference_video_frames[
+ clip_denoising_metadata.frame_start_index : clip_denoising_metadata.frame_end_index
+ ]
+ )
+ reference_video_pixels = rearrange(
+ torch.from_numpy(reference_video_clip_np).to(device), "t h w c -> c t h w"
+ )
+ # [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, t, h, w]
+ reference_video_pixels = (reference_video_pixels.float() / 127.5 - 1).to(
+ torch.bfloat16
+ )
+
+ # [16, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] fp32
+ # Stays fp32 (vae.encode returns .float()); forward_ref cats it with the
+ # bf16 reference_video_condition under autocast type promotion.
+ reference_video_latents = vae_encoder([reference_video_pixels])[0]
+ (
+ _,
+ reference_video_latent_t,
+ reference_video_latent_h,
+ reference_video_latent_w,
+ ) = reference_video_latents.shape
+ reference_video_grid_sizes = (
+ reference_video_latent_t,
+ reference_video_latent_h // 2,
+ reference_video_latent_w // 2,
+ )
+
+ # [3, reference_video_height, reference_video_width] bf16
+ reference_video_frame_0 = reference_video_pixels[:, 0]
+ reference_video_frame_0_image_embeddings = image_embedder(
+ [reference_video_frame_0[:, None, :, :]]
+ ).to(torch.bfloat16) # [1, 257, 1280] bf16
+
+ # reference_video_condition (forward_ref): mask, then VAE latent of the reference-video slice.
+ # Every frame of the slice is given, so the latent is reference_video_latents itself;
+ # the official code encodes the same pixels a second time here.
+ reference_video_condition_mask = make_conditioning_mask(
+ reference_video_latent_t,
+ reference_video_latent_h,
+ reference_video_latent_w,
+ num_frames,
+ device=device,
+ )
+ # [20, reference_video_latent_t, reference_video_latent_h, reference_video_latent_w] bf16
+ reference_video_condition = torch.concat(
+ [reference_video_condition_mask, reference_video_latents]
+ ).to(torch.bfloat16)
+
+ # Drawn from the request's seeded generator; clips are built in schedule order, so
+ # clip k always gets the k-th draw and (seed, clip_index) identifies the noise. Clip 0's
+ # draw already happened in the before-denoising stage (it is batch.latents).
+ if (
+ clip_denoising_metadata.clip_index == 0
+ and request_state.clip_0_init_noise is not None
+ ):
+ init_noise = request_state.clip_0_init_noise
+ else:
+ init_noise = torch.randn(
+ *init_noise_shape,
+ dtype=torch.float32,
+ device=device,
+ generator=request_state.generator,
+ )
+
+ return WanAnimate2ClipConditioning(
+ reference_video_latents=reference_video_latents,
+ generation_video_grid_sizes=grid_sizes,
+ reference_video_frame_0_image_embeddings=reference_video_frame_0_image_embeddings,
+ reference_video_condition=reference_video_condition,
+ prompt_ref_embeddings=request_state.prompt_ref_embeddings,
+ generation_condition=generation_condition,
+ reference_video_grid_sizes=reference_video_grid_sizes,
+ full_clip_grid_sizes=request_state.full_clip_grid_sizes,
+ num_frames=clip_denoising_metadata.num_frames,
+ init_noise=init_noise,
+ )
+
+
+class WanAnimate2BeforeDenoisingStage(TextEncodingStage):
+ """Native text encoding plus the model-specific reference and clip conditioning."""
+
+ # Text-only stage dedup must not skip fresh image/video conditions or the request RNG.
+ deduplicated_output_fields = ()
+
+ def __init__(
+ self,
+ vae: WanAnimate2VaeAdapter,
+ image_encoder: WanAnimate2ImageEncoderAdapter,
+ text_encoder: torch.nn.Module,
+ tokenizer,
+ pipeline_config: Wan_Animate_2_14B_Config,
+ scheduler: DPMSolverMultistepScheduler,
+ ) -> None:
+ super().__init__(text_encoders=[text_encoder], tokenizers=[tokenizer])
+ self.vae = vae
+ self.image_encoder = image_encoder
+ self.pipeline_config = pipeline_config
+ self._silent_reference_logged = False
+ self._audio_failure_logged = False
+ # Shared with the denoising stage, which resets it per clip; used here only to derive
+ # batch.timesteps on the same sigma grid.
+ self.scheduler = scheduler
+
+ def component_uses(
+ self, server_args: ServerArgs, stage_name: str | None = None
+ ) -> list[ComponentUse]:
+ name = self._component_stage_name(stage_name)
+ return super().component_uses(server_args, stage_name) + [
+ ComponentUse(stage_name=name, component_name="vae"),
+ ComponentUse(stage_name=name, component_name="image_encoder"),
+ ]
+
+ def forward(self, batch: Req, server_args: ServerArgs) -> Req:
+ device = get_local_torch_device()
+ sp_size = max(1, int(get_sp_world_size()))
+
+ inputs = _WanAnimate2Inputs.from_request(batch)
+
+ # One seeded generator per request; clips draw their noise from it in order.
+ generator = torch.Generator(device=device).manual_seed(inputs.seed)
+
+ audio, audio_sample_rate = self._extract_audio(inputs)
+ request_state = self._build_request_state(
+ inputs,
+ device,
+ sp_size,
+ generator,
+ server_args,
+ audio=audio,
+ audio_sample_rate=audio_sample_rate,
+ )
+ if len(request_state.schedule) > 1:
+ self.log_info(
+ "reference video splits into %d clips; each is conditioned in the denoising "
+ "loop via build_clip_conditioning.",
+ len(request_state.schedule),
+ )
+ batch.extra[WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY] = request_state
+ _mirror_request_state_into_batch(
+ batch,
+ request_state,
+ scheduler=self.scheduler,
+ flow_shift=self.pipeline_config.flow_shift,
+ )
+ return batch
+
+ def _build_request_state(
+ self,
+ inputs: _WanAnimate2Inputs,
+ device: torch.device,
+ sp_size: int,
+ generator: torch.Generator,
+ server_args: ServerArgs,
+ *,
+ audio: torch.Tensor | None,
+ audio_sample_rate: int | None,
+ ) -> WanAnimate2RequestState:
+ width, height = inputs.width, inputs.height
+
+ reference_image = cv2.imread(inputs.reference_image_path)
+ if reference_image is None:
+ raise ValueError(
+ f"Wan-Animate-2: failed to read reference image at {inputs.reference_image_path!r} "
+ "(cv2.imread returned None: missing file or unsupported/corrupt image)."
+ )
+ reference_image = reference_image[..., ::-1] # BGR -> RGB
+ reference_image, letterbox_info = resize_by_area(
+ reference_image, width * height, divisor=16
+ )
+
+ # reference video: decode, fps resample, zigzag pad.
+ frames = read_reference_video_frames(inputs.reference_video_path, inputs.fps)
+ reference_video_frames = [
+ resize_by_area(f, width * height, divisor=16)[0] for f in frames
+ ]
+ num_reference_video_frames = len(frames)
+ target_len = get_padding_len(num_reference_video_frames, inputs.clip_len)
+ reference_video_frames = zigzag_padding(reference_video_frames, target_len)
+
+ # Keep the official padding and geometry of every needed clip, but do not
+ # denoise clips whose entire output would be discarded by the final trim.
+ schedule = [
+ clip
+ for clip in build_schedule(
+ reference_video_frames, inputs.clip_len, inputs.num_frames_conditioning
+ )
+ if clip.frame_start_index + clip.num_frames_conditioning_for_clip
+ < num_reference_video_frames
+ ]
+
+ # reference-image encodings (constant across clips)
+ reference_image_height, reference_image_width = (
+ reference_image.shape[0],
+ reference_image.shape[1],
+ )
+ latent_h, latent_w = reference_image_height // 8, reference_image_width // 8
+
+ reference_image_pixel_values = rearrange(
+ torch.from_numpy(reference_image).to(device), "h w c -> c 1 h w"
+ )
+ # [0, 255] uint8 -> [-1.0, 1.0] bf16; [3, 1, H, W]
+ reference_image_pixel_values = (
+ reference_image_pixel_values.float() / 127.5 - 1
+ ).to(torch.bfloat16)
+
+ with (
+ torch.autocast(device_type=device.type, dtype=torch.bfloat16),
+ torch.no_grad(),
+ ):
+ # [16, 1, latent_h, latent_w] fp32
+ reference_image_latents = self.vae_encoder([reference_image_pixel_values])[
+ 0
+ ]
+
+ # [4, 1, latent_h, latent_w] fp32
+ reference_image_mask = make_conditioning_mask(
+ 1, latent_h, latent_w, 1, device=device
+ )
+
+ # [20, 1, latent_h, latent_w] bf16
+ reference_image_condition = torch.concat(
+ [reference_image_mask, reference_image_latents]
+ ).to(torch.bfloat16)
+
+ reference_image_embeddings = self.image_embedder(
+ [reference_image_pixel_values]
+ ).to(torch.bfloat16) # [1, 257, 1280] bf16
+
+ # Keep batch-one encoder arithmetic while sharing native caching and residency.
+ prompt_embeddings, prompt_ref_embeddings, negative_prompt_embeddings = [
+ self.encode_text(prompt, server_args, device=device)[0][0]
+ for prompt in (inputs.prompt, inputs.prompt_ref, inputs.negative_prompt)
+ ]
+
+ return WanAnimate2RequestState(
+ device=device,
+ sp_size=sp_size,
+ generator=generator,
+ inputs=inputs,
+ letterbox_info=letterbox_info,
+ reference_image_height=reference_image_height,
+ reference_image_width=reference_image_width,
+ latent_h=latent_h,
+ latent_w=latent_w,
+ reference_image_condition=reference_image_condition,
+ reference_image_embeddings=reference_image_embeddings,
+ prompt_embeddings=prompt_embeddings,
+ negative_prompt_embeddings=negative_prompt_embeddings,
+ prompt_ref_embeddings=prompt_ref_embeddings,
+ reference_video_frames=reference_video_frames,
+ schedule=schedule,
+ num_reference_video_frames=num_reference_video_frames,
+ full_clip_grid_sizes=generation_token_grid(
+ inputs.clip_len, latent_h, latent_w
+ ),
+ audio=audio,
+ audio_sample_rate=audio_sample_rate,
+ )
+
+ def _extract_audio(
+ self, inputs: _WanAnimate2Inputs
+ ) -> tuple[torch.Tensor | None, int | None]:
+ """Per-request opt-out; any failure yields (None, None), a silent video."""
+ if not inputs.enable_audio:
+ return None, None
+ return extract_reference_video_audio(
+ inputs.reference_video_path,
+ log_info=self.log_info,
+ log_no_audio_track=self._log_silent_reference,
+ log_warning=self._log_audio_failure,
+ )
+
+ def _log_silent_reference(self, msg: str, *args: Any) -> None:
+ # A reference video without an audio track is normal input, not a failure:
+ # one info line per process, then debug.
+ if self._silent_reference_logged:
+ self.log_debug(msg, *args)
+ return
+ self._silent_reference_logged = True
+ self.log_info(
+ msg + " Set enable_audio=false on the request to skip audio extraction.",
+ *args,
+ )
+
+ def _log_audio_failure(self, msg: str, *args: Any) -> None:
+ # A track the audio reader cannot decode is the same for every request: warn once
+ # per process with the remedy, then keep the detail at debug.
+ if self._audio_failure_logged:
+ self.log_debug(msg, *args)
+ return
+ self._audio_failure_logged = True
+ self.log_warning(
+ msg
+ + " Set enable_audio=false on the request to skip audio extraction; further "
+ "audio failures are logged at debug level.",
+ *args,
+ )
+
+ def vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
+ with self.use_declared_component(component_name="vae", module=self.vae.vae):
+ return self.vae.encode(videos)
+
+ def image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor:
+ with self.use_declared_component(
+ component_name="image_encoder", module=self.image_encoder.model
+ ):
+ return self.image_encoder.visual(videos)
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py
new file mode 100644
index 0000000000..a816fa71c3
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/denoising.py
@@ -0,0 +1,532 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Wan-Animate-2 denoising and output stages.
+
+WanAnimate2DenoisingStage orchestrates clips around the shared denoising loop: the in-context DiT
+takes per-clip conditioning via its ``clip_cond`` kwarg and the clip's reference K/V via
+``reference_kv`` (built once per clip with ``build_reference_kv`` and dropped with the clip),
+runs cond/uncond CFG as two forwards (the unconditional branch skips block 9 via
+``is_unconditional``), resets the scheduler per clip on the official sigma grid, and
+VAE-decodes each clip in-loop to feed the next.
+WanAnimate2OutputStage then emits the assembled frames.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field, replace
+from typing import TYPE_CHECKING
+
+import numpy as np
+import torch
+from einops import rearrange
+from torch import nn
+
+from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
+from sglang.multimodal_gen.runtime.distributed import (
+ get_local_torch_device,
+ model_parallel_is_initialized,
+)
+from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import (
+ run_cfg_parallel,
+)
+from sglang.multimodal_gen.runtime.distributed.cfg_policy import (
+ CFGBranch,
+ CFGPolicy,
+)
+from sglang.multimodal_gen.runtime.distributed.parallel_state import (
+ get_classifier_free_guidance_world_size,
+)
+from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
+from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
+ ComponentUse,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
+ WanAnimate2ClipConditioning,
+ WanAnimate2ReferenceKV,
+)
+from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
+ DPMSolverMultistepScheduler,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
+from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
+from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
+from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import (
+ DenoisingContext,
+ DenoisingStage,
+ DenoisingStepState,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
+ WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY,
+ WanAnimate2RequestState,
+ build_clip_conditioning,
+ get_sampling_sigmas,
+ request_state_from_batch,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
+ WanAnimate2ImageEncoderAdapter,
+ WanAnimate2VaeAdapter,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
+ LetterboxInfo,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
+ V,
+ VerificationResult,
+)
+from sglang.multimodal_gen.runtime.platforms import current_platform
+from sglang.multimodal_gen.runtime.server_args import ServerArgs
+from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
+
+logger = init_logger(__name__)
+
+if TYPE_CHECKING:
+ from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
+ ComposedPipelineBase,
+ )
+
+
+def _is_request_state(value: object) -> bool:
+ return isinstance(value, WanAnimate2RequestState)
+
+
+def _is_decoded_frames(value: object) -> bool:
+ """``[T, H, W, C]`` uint8 frames assembled by the denoising stage."""
+ return isinstance(value, np.ndarray) and value.ndim == 4 and value.dtype == np.uint8
+
+
+@dataclass(kw_only=True)
+class WanAnimate2DenoisingContext(DenoisingContext):
+ clip_condition: WanAnimate2ClipConditioning
+ reference_image_embeddings: torch.Tensor
+ cfg_parallel: bool
+ guidance_scale: float
+ reference_kv: WanAnimate2ReferenceKV | None = None
+ context_by_branch: dict[str, torch.Tensor] = field(default_factory=dict)
+
+
+class WanAnimate2DenoisingStage(DenoisingStage):
+ """Wan-Animate-2 in-context denoising over all reference-video clips.
+
+ Clip 0 uses the precomputed conditioning; clips > 0 are built from the previous clip's decoded
+ frames, denoised, decoded in-loop, overlap-trimmed, and assembled into the full video.
+ """
+
+ def __init__(
+ self,
+ *,
+ transformer: nn.Module,
+ scheduler: DPMSolverMultistepScheduler,
+ pipeline: ComposedPipelineBase,
+ vae: WanAnimate2VaeAdapter,
+ image_encoder: WanAnimate2ImageEncoderAdapter,
+ ) -> None:
+ super().__init__(
+ transformer=transformer, scheduler=scheduler, pipeline=pipeline, vae=vae
+ )
+ self.vae: WanAnimate2VaeAdapter = vae
+ self.clip_decoder = DecodingStage(vae, pipeline=pipeline)
+ # Clips > 0 are conditioned inside this stage, so it holds the image encoder too.
+ self.image_encoder = image_encoder
+
+ def _vae_encoder(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
+ with self.use_declared_component(component_name="vae", module=self.vae.vae):
+ return self.vae.encode(videos)
+
+ def _image_embedder(self, videos: list[torch.Tensor]) -> torch.Tensor:
+ with self.use_declared_component(
+ component_name="image_encoder", module=self.image_encoder.model
+ ):
+ return self.image_encoder.visual(videos)
+
+ def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
+ result = VerificationResult()
+ result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.min_dims(1)])
+ result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(4)])
+ result.add_check("raw_latent_shape", batch.raw_latent_shape, V.not_none)
+ result.add_check("sigmas", batch.sigmas, V.list_not_empty)
+ result.add_check(
+ "prompt_embeds", batch.prompt_embeds, V.list_of_tensors_dims(2)
+ )
+ result.add_check(
+ "negative_prompt_embeds",
+ batch.negative_prompt_embeds,
+ V.list_of_tensors_dims(2),
+ )
+ result.add_check("image_embeds", batch.image_embeds, V.list_of_tensors_dims(3))
+ result.add_check(
+ "num_inference_steps", batch.num_inference_steps, V.positive_int
+ )
+ result.add_check("guidance_scale", batch.guidance_scale, V.non_negative_float)
+ result.add_check("generator", batch.generator, V.generator_or_list_generators)
+ result.add_check(
+ WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY,
+ batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY),
+ _is_request_state,
+ )
+ return result
+
+ def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
+ result = VerificationResult()
+ request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
+ result.add_check(
+ "decoded_frames",
+ (
+ request_state.decoded_frames
+ if isinstance(request_state, WanAnimate2RequestState)
+ else None
+ ),
+ _is_decoded_frames,
+ )
+ return result
+
+ def component_uses(
+ self, server_args: ServerArgs, stage_name: str | None = None
+ ) -> list[ComponentUse]:
+ # build_clip_conditioning runs CLIP.visual inside this stage, so image_encoder must stay
+ # resident (it is CPU-offloaded by default for video tasks).
+ uses = super().component_uses(server_args, stage_name)
+ # The VAE adapter runs the VAE in fp32 with autocast off; do not let the manager
+ # cast it to the configured vae precision on the way in.
+ uses = [
+ replace(use, target_dtype=None) if use.component_name == "vae" else use
+ for use in uses
+ ]
+ if not any(use.component_name == "image_encoder" for use in uses):
+ uses.append(
+ ComponentUse(
+ stage_name=self._component_stage_name(stage_name),
+ component_name="image_encoder",
+ )
+ )
+ return uses
+
+ @torch.no_grad()
+ def _denoise_single_clip(
+ self,
+ batch: Req,
+ server_args: ServerArgs,
+ request_state: WanAnimate2RequestState,
+ clip_condition: WanAnimate2ClipConditioning,
+ ) -> torch.Tensor:
+ """Denoise one clip; return the latents with the reference-image slot dropped.
+ ``batch`` is handed to set_forward_context, which the attention backends read, and
+ supplies the metrics object the per-step records go to."""
+ device = get_local_torch_device()
+
+ prompt_embeddings = request_state.prompt_embeddings
+ negative_prompt_embeddings = request_state.negative_prompt_embeddings
+ reference_image_embeddings = request_state.reference_image_embeddings
+
+ # Official sampler grid: sigma_0 is exactly 1.0, so sigmas are set explicitly.
+ # set_timesteps also resets all solver state for the new clip.
+ num_inference_steps = request_state.inputs.num_inference_steps
+ sample_shift = server_args.pipeline_config.flow_shift
+ self.scheduler.set_timesteps(
+ sigmas=get_sampling_sigmas(num_inference_steps, sample_shift), device=device
+ )
+
+ guidance_scale = request_state.inputs.guidance_scale
+
+ # negative_prompt_embeddings is always computed (an unset prompt encodes ""), so CFG
+ # depends on the scale alone, unlike Req.validate which also needs a negative prompt.
+ perform_cfg = guidance_scale > 1
+
+ # CFG parallel: cond on cfg_rank 0, uncond on cfg_rank 1, then all-gather. Only
+ # with --enable-cfg-parallel and initialized model-parallel groups.
+ cfg_parallel = (
+ perform_cfg
+ and bool(server_args.enable_cfg_parallel)
+ and model_parallel_is_initialized()
+ )
+ if cfg_parallel and get_classifier_free_guidance_world_size() != 2:
+ logger.warning_once(
+ "CFG parallel enabled but the CFG group world size is "
+ f"{get_classifier_free_guidance_world_size()} (Wan-Animate-2 expects 2 "
+ "for its cond/uncond split); using the sequential two-pass CFG path."
+ )
+ cfg_parallel = False
+ branches = [
+ CFGBranch(
+ "conditional",
+ True,
+ {
+ "encoder_hidden_states": prompt_embeddings,
+ "is_unconditional": False,
+ },
+ )
+ ]
+ if perform_cfg:
+ branches.append(
+ CFGBranch(
+ "unconditional",
+ False,
+ {
+ "encoder_hidden_states": negative_prompt_embeddings,
+ "is_unconditional": True,
+ },
+ )
+ )
+ cfg_policy = CFGPolicy(branches=branches)
+
+ ctx = WanAnimate2DenoisingContext(
+ scheduler=self.scheduler,
+ extra_step_kwargs={},
+ target_dtype=torch.bfloat16,
+ autocast_enabled=True,
+ timesteps=self.scheduler.timesteps,
+ num_inference_steps=num_inference_steps,
+ num_warmup_steps=0,
+ image_kwargs={},
+ pos_cond_kwargs={},
+ neg_cond_kwargs={},
+ latents=clip_condition.init_noise,
+ boundary_timestep=None,
+ z=None,
+ reserved_frames_mask=None,
+ seq_len=None,
+ guidance=None,
+ is_warmup=batch.is_warmup,
+ cfg_policy=cfg_policy,
+ # DPM-Solver advances once per timestep, irrespective of solver order
+ extra={"progress_step_interval": 1},
+ clip_condition=clip_condition,
+ reference_image_embeddings=reference_image_embeddings,
+ cfg_parallel=cfg_parallel,
+ guidance_scale=guidance_scale,
+ )
+ # clip trajectories are not part of the assembled-video output contract
+ self._run_denoising_loop(ctx, batch, server_args, collect_trajectory=False)
+ return ctx.latents[:, 1:]
+
+ def _before_denoising_loop(
+ self, ctx: WanAnimate2DenoisingContext, batch: Req, server_args: ServerArgs
+ ) -> None:
+ # activate the DiT before the reference pass, not just the first denoise step
+ self.begin_declared_component_use(
+ component_name="transformer", module=self.transformer
+ )
+ with set_forward_context(
+ current_timestep=0, attn_metadata=None, forward_batch=batch
+ ):
+ ctx.reference_kv = self.transformer.build_reference_kv(
+ clip_cond=ctx.clip_condition
+ )
+
+ def _prepare_step_state(
+ self,
+ ctx: WanAnimate2DenoisingContext,
+ batch: Req,
+ server_args: ServerArgs,
+ step_index: int,
+ t_host: torch.Tensor,
+ timesteps_cpu: torch.Tensor,
+ ) -> DenoisingStepState:
+ return DenoisingStepState(
+ step_index=step_index,
+ t_host=t_host,
+ t_device=ctx.timesteps[step_index],
+ t_int=int(t_host.item()),
+ current_model=self.transformer,
+ current_guidance_scale=ctx.guidance_scale,
+ attn_metadata=None,
+ )
+
+ def _run_denoising_step(
+ self,
+ ctx: WanAnimate2DenoisingContext,
+ step: DenoisingStepState,
+ batch: Req,
+ server_args: ServerArgs,
+ ) -> None:
+ def predict(branch: CFGBranch) -> torch.Tensor:
+ with set_forward_context(
+ current_timestep=step.step_index,
+ attn_metadata=None,
+ forward_batch=batch,
+ ):
+ if branch.name not in ctx.context_by_branch:
+ ctx.context_by_branch[branch.name] = (
+ self.transformer.prepare_context(
+ branch.kwargs["encoder_hidden_states"],
+ ctx.reference_image_embeddings,
+ )
+ )
+ prediction = self.transformer(
+ hidden_states=ctx.latents,
+ timestep=step.t_device.unsqueeze(0),
+ encoder_hidden_states_image=ctx.reference_image_embeddings,
+ clip_cond=ctx.clip_condition,
+ reference_kv=ctx.reference_kv,
+ projected_context=ctx.context_by_branch[branch.name],
+ **branch.kwargs,
+ )
+ return prediction.contiguous() if ctx.cfg_parallel else prediction
+
+ predictions = (
+ run_cfg_parallel(ctx.cfg_policy, predict)
+ if ctx.cfg_parallel
+ else [predict(branch) for branch in ctx.cfg_policy.branches]
+ )
+ noise_pred = predictions[0]
+ if len(predictions) == 2:
+ noise_pred = predictions[1] + step.current_guidance_scale * (
+ noise_pred - predictions[1]
+ )
+ ctx.latents = ctx.scheduler.step(
+ noise_pred,
+ step.t_device,
+ ctx.latents.unsqueeze(0),
+ return_dict=False,
+ )[0].squeeze(0)
+
+ @torch.no_grad()
+ def forward(self, batch: Req, server_args: ServerArgs) -> Req:
+ request_state = request_state_from_batch(batch)
+
+ output_frames: list[np.ndarray] = [] # [H, W, C] uint8 frames
+
+ # [C, num_frames_conditioning, H, W] bf16 in [-1, 1]
+ prev_clip_conditioning_frames: torch.Tensor | None = None
+
+ # The DiT runs in bf16 autocast like upstream Wan (timestep and norm math in fp32).
+ with torch.autocast(
+ device_type=current_platform.device_type,
+ dtype=torch.bfloat16,
+ enabled=True,
+ ):
+ for clip_denoising_metadata in request_state.schedule:
+ is_clip_0 = clip_denoising_metadata.clip_index == 0
+
+ if not is_clip_0 and prev_clip_conditioning_frames is None:
+ raise ValueError(
+ "For all clips except clip-0 `prev_clip_conditioning_frames` must be not None."
+ )
+ clip_condition = build_clip_conditioning(
+ request_state,
+ clip_denoising_metadata,
+ prev_clip_conditioning_frames,
+ vae_encoder=self._vae_encoder,
+ image_embedder=self._image_embedder,
+ )
+
+ latents = self._denoise_single_clip(
+ batch,
+ server_args,
+ request_state,
+ clip_condition=clip_condition,
+ )
+
+ denoised_latents = latents.to(dtype=torch.float32)
+ with self.use_declared_component(
+ component_name="vae", module=self.vae.vae
+ ):
+ # [1, C, T, H, W] fp32 in [-1, 1]
+ decoded_frames = self.clip_decoder.decode_raw(
+ denoised_latents.unsqueeze(0),
+ server_args,
+ vae_dtype=torch.float32,
+ )
+
+ # [T, H, W, C] uint8
+ output_frames_for_clip = (
+ rearrange(((decoded_frames + 1) * 127.5), "1 c t h w -> t h w c")
+ .detach()
+ .to(torch.uint8)
+ .cpu()
+ .numpy()
+ )
+
+ # Drop the overlap frames the previous clip already emitted; 0 for clip 0.
+ output_frames_for_clip = output_frames_for_clip[
+ clip_denoising_metadata.num_frames_conditioning_for_clip :
+ ]
+ output_frames.extend(output_frames_for_clip)
+
+ # The next clip takes the last ``num_frames_conditioning`` frames.
+ prev_clip_conditioning_frames = (
+ decoded_frames[
+ 0, :, -request_state.inputs.num_frames_conditioning :
+ ]
+ .detach()
+ .to(torch.bfloat16)
+ )
+ # only the independent overlap frames survive into the next clip
+ del decoded_frames, denoised_latents, latents, clip_condition
+
+ if torch.cuda.is_available():
+ torch.cuda.empty_cache()
+
+ # Already-decoded [T, H, W, C] uint8; WanAnimate2OutputStage must not re-decode.
+ request_state.decoded_frames = self._post_process_frames(
+ output_frames,
+ request_state.num_reference_video_frames,
+ request_state.letterbox_info,
+ )
+ return batch
+
+ @staticmethod
+ def _post_process_frames(
+ frames: list[np.ndarray],
+ num_output_frames: int,
+ letterbox_info: LetterboxInfo,
+ ) -> np.ndarray:
+ """Stack ``list[np.ndarray]`` to ``[T, H, W, C]`` uint8, trim to ``num_output_frames`` and crop the
+ ``resize_by_area`` letterbox."""
+ if len(frames) == 0:
+ raise ValueError(
+ "WanAnimate2DenoisingStage produced 0 decoded frames to post-process."
+ )
+ frames_np = np.stack(frames, axis=0) # [T, H, W, C]
+ frames_np = frames_np[:num_output_frames]
+
+ return letterbox_info.crop(frames_np)
+
+
+class WanAnimate2OutputStage(PipelineStage):
+ """Terminal stage: wrap the already-decoded, assembled frames in an OutputBatch.
+
+ Replaces the standard DecodingStage (which would VAE-decode a single-clip latent).
+ """
+
+ @property
+ def role_affinity(self) -> RoleType:
+ # The terminal slot of the standard stage layout; disaggregated deployment itself
+ # is rejected by Wan_Animate_2_14B_Config.supports_disaggregation.
+ return RoleType.DECODER
+
+ def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
+ result = VerificationResult()
+ request_state = batch.extra.get(WAN_ANIMATE_2_REQUEST_STATE_EXTRA_KEY)
+ result.add_check(
+ "decoded_frames",
+ (
+ request_state.decoded_frames
+ if isinstance(request_state, WanAnimate2RequestState)
+ else None
+ ),
+ _is_decoded_frames,
+ )
+ return result
+
+ def verify_output(
+ self, batch: OutputBatch, server_args: ServerArgs
+ ) -> VerificationResult:
+ result = VerificationResult()
+ result.add_check("output", batch.output, V.list_not_empty)
+ return result
+
+ @torch.no_grad()
+ def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
+ request_state = request_state_from_batch(batch)
+ frames = request_state.decoded_frames
+ if frames is None:
+ raise ValueError(
+ "WanAnimate2OutputStage: WanAnimate2DenoisingStage did not assemble decoded frames."
+ )
+ # frames: [T, H, W, C] uint8, consumed directly by the save path. Audio, when
+ # extracted, is [1, C, L] fp32 in [-1, 1] with its sample rate; None gives a silent mp4.
+ return OutputBatch(
+ output=[frames],
+ audio=request_state.audio,
+ audio_sample_rate=request_state.audio_sample_rate,
+ metrics=batch.metrics,
+ )
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py
new file mode 100644
index 0000000000..e1457575e1
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/encoder_adapters.py
@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# Adapters exposing the official Wan encoder API over the diffusers/transformers models the
+# sglang loaders produce: image_encoder.visual takes list[torch.Tensor] of [C, T, H, W] and returns
+# [B, 257, 1280]; vae.encode/decode fold in the Wan latent scaling.
+from __future__ import annotations
+
+from typing import Any
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import CLIPImageProcessor
+
+from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
+
+
+class WanAnimate2ImageEncoderAdapter:
+ """Official Wan CLIP API over a ``CLIPVisionModel``: ``visual`` takes ``list[torch.Tensor]`` of ``[C, T, H, W]``
+ in [-1, 1] and returns ``hidden_states[-2]`` as ``[B, 257, 1280]`` in the model dtype, the pre-final-LayerNorm
+ slice the Wan-I2V ``postprocess_image`` consumes."""
+
+ def __init__(
+ self,
+ model: nn.Module,
+ image_size: int,
+ image_mean: tuple[float, float, float],
+ image_std: tuple[float, float, float],
+ hidden_state_index: int = -2,
+ device: torch.device | str | None = None,
+ ) -> None:
+ self.model = model
+ self.image_size = image_size
+ self.image_mean = image_mean
+ self.image_std = image_std
+ self.hidden_state_index = hidden_state_index
+ self.device = device
+
+ @classmethod
+ def from_loaded(
+ cls,
+ model: nn.Module,
+ image_processor: CLIPImageProcessor,
+ device: torch.device | str | None = None,
+ ) -> WanAnimate2ImageEncoderAdapter:
+ """Build the adapter with image_size / mean / std from the checkpoint's
+ ``CLIPImageProcessor`` (``crop_size`` is the post-resize crop the model sees)."""
+ crop_h = int(image_processor.crop_size["height"])
+ crop_w = int(image_processor.crop_size["width"])
+ if crop_h != crop_w:
+ raise ValueError(
+ "WanAnimate2ImageEncoderAdapter expects a square CLIP crop_size; got "
+ f"height={crop_h}, width={crop_w}."
+ )
+ return cls(
+ model,
+ image_size=crop_h,
+ image_mean=tuple(image_processor.image_mean),
+ image_std=tuple(image_processor.image_std),
+ device=device,
+ )
+
+ def _resolve_device(self) -> torch.device:
+ if self.device is not None:
+ return torch.device(self.device)
+ return next(self.model.parameters()).device
+
+ @torch.no_grad()
+ def visual(self, videos: list[torch.Tensor]) -> torch.Tensor:
+ # The caller holds the component through the residency manager, so the model is
+ # already where it should be; inputs follow it. Runs in the model's own dtype with
+ # autocast off: the fused patch-embed conv does not autocast an fp32 weight against
+ # a bf16 input.
+ device = self._resolve_device()
+ model_dtype = next(self.model.parameters()).dtype
+ size = (self.image_size, self.image_size)
+
+ # Each video [C, T, H, W] becomes T images [T, C, image_size, image_size]; the images
+ # of all videos are stacked into one batch [N, C, image_size, image_size], N = total
+ # frame count.
+ pixels = torch.cat(
+ [
+ F.interpolate(
+ video.transpose(0, 1).to(device=device, dtype=model_dtype),
+ size=size,
+ mode="bicubic",
+ align_corners=False,
+ )
+ for video in videos
+ ]
+ )
+
+ # [-1, 1] -> [0, 1] then Normalize(mean, std), in the model's own dtype.
+ mean = torch.tensor(self.image_mean, device=device, dtype=pixels.dtype).view(
+ 1, 3, 1, 1
+ )
+ std = torch.tensor(self.image_std, device=device, dtype=pixels.dtype).view(
+ 1, 3, 1, 1
+ )
+ pixels = pixels.mul(0.5).add(0.5).sub(mean).div(std)
+
+ # Encode; return the hidden_states[-2] slice.
+ with (
+ torch.autocast(device_type=device.type, enabled=False),
+ set_forward_context(current_timestep=0, attn_metadata=None),
+ ):
+ outputs = self.model(pixel_values=pixels, output_hidden_states=True)
+ return outputs.hidden_states[self.hidden_state_index]
+
+
+class WanAnimate2VaeAdapter(nn.Module):
+ """Stage-local latent-scaling adapter over the pipeline's native ``AutoencoderKLWan``.
+ ``encode`` takes ``list[torch.Tensor]`` (or ``[B, C, T, H, W]``) and returns ``list[torch.Tensor]`` of
+ ``[16, T', H', W']`` fp32, ``(mode() - latents_mean) / latents_std``; ``decode`` inverts that and returns
+ ``[B, C, T, H, W]`` fp32 in [-1, 1], which is why ``Wan_Animate_2_14B_Config.get_decode_scale_and_shift``
+ returns ``(1.0, None)``."""
+
+ @property
+ def use_parallel_decode(self) -> bool:
+ return self.vae.use_parallel_decode
+
+ def enable_tiling(self) -> None:
+ self.vae.enable_tiling()
+
+ def __init__(self, vae: nn.Module) -> None:
+ super().__init__()
+ if not isinstance(vae, nn.Module):
+ raise TypeError(
+ "WanAnimate2VaeAdapter expects an nn.Module AutoencoderKLWan; got "
+ f"{type(vae).__name__!r}."
+ )
+ self.vae = vae
+
+ # Per-channel latent scaling (length z_dim), which the sglang AutoencoderKLWan
+ # copies from its config onto the module. Plain (unregistered) tensors, reshaped
+ # to broadcast over [B, C, T, H, W].
+ self._latents_mean = torch.as_tensor(
+ vae.latents_mean, dtype=torch.float32
+ ).view(1, -1, 1, 1, 1)
+ self._latents_std = torch.as_tensor(vae.latents_std, dtype=torch.float32).view(
+ 1, -1, 1, 1, 1
+ )
+
+ @staticmethod
+ def _to_list_of_tensors(
+ items: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
+ ) -> list[torch.Tensor]:
+ """Per-item tensors from either a list or a batched ``[B, ...]`` tensor; the stages
+ pass lists, the shared decode path passes a batch."""
+ if isinstance(items, (list, tuple)):
+ return list(items)
+ return [items[i] for i in range(items.shape[0])]
+
+ def _mean_std_like(self, input: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ return (
+ self._latents_mean.to(device=input.device, dtype=input.dtype),
+ self._latents_std.to(device=input.device, dtype=input.dtype),
+ )
+
+ @torch.no_grad()
+ def encode(
+ self,
+ videos: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
+ *args: Any,
+ **kwargs: Any,
+ ) -> list[torch.Tensor]:
+ out: list[torch.Tensor] = []
+ for sample in self._to_list_of_tensors(videos):
+ # Run fp32 with autocast off: the fused causal-conv path has no dtype guard, so
+ # an outer autocast(bf16) would cast the conv input against the fp32 bias.
+ # The sglang AutoencoderKLWan returns the DiagonalGaussianDistribution itself,
+ # not a diffusers AutoencoderKLOutput.
+ with torch.autocast(device_type=sample.device.type, enabled=False):
+ posterior = self.vae.encode(sample.unsqueeze(0).float())
+
+ # Deterministic mu (not a sampled draw).
+ mu = posterior.mode().float()
+ mean, std = self._mean_std_like(mu)
+ z = (mu - mean) / std
+ out.append(z.squeeze(0))
+ return out
+
+ @torch.no_grad()
+ def decode(
+ self,
+ latents: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
+ *args: Any,
+ **kwargs: Any,
+ ) -> torch.Tensor:
+ out: list[torch.Tensor] = []
+ for sample in self._to_list_of_tensors(latents):
+ z = sample.unsqueeze(0).float()
+ mean, std = self._mean_std_like(z)
+ z = z * std + mean
+
+ # fp32, autocast OFF: see ``encode`` above. decode returns a plain tensor.
+ with torch.autocast(device_type=z.device.type, enabled=False):
+ decoded = self.vae.decode(z)
+ out.append(decoded.float().clamp_(-1.0, 1.0).squeeze(0))
+
+ # Clips are decoded individually; avoid copying their full pixel tensor.
+ return out[0].unsqueeze(0) if len(out) == 1 else torch.stack(out, dim=0)
+
+ def forward(
+ self,
+ z: torch.Tensor | list[torch.Tensor] | tuple[torch.Tensor, ...],
+ *args: Any,
+ **kwargs: Any,
+ ) -> torch.Tensor:
+ return self.decode(z, *args, **kwargs)
diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py
new file mode 100644
index 0000000000..6785b76141
--- /dev/null
+++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/wan_animate_2/preprocess.py
@@ -0,0 +1,324 @@
+# SPDX-License-Identifier: Apache-2.0
+# Adapted from the Wan-Animate-2 reference implementation (Apache-2.0):
+# https://github.com/Wan-Video/Wan-Animate-2
+#
+# Keep the original Wan preprocessing numerics; the newer Diffusers modular
+# processor is not a numerically interchangeable replacement. The
+# zigzag_padding / resize_by_area names are kept for parity with the official helpers;
+# make_conditioning_mask is the official get_i2v_mask.
+from __future__ import annotations
+
+import math
+from copy import deepcopy
+from io import BytesIO
+from typing import Literal
+
+import cv2
+import msgspec
+import numpy as np
+import torch
+
+from sglang.srt.utils.common import get_video_bytes
+
+__all__ = [
+ "LetterboxInfo",
+ "MIN_SINGLE_CLIP_LEN",
+ "resize_by_area",
+ "letterbox_resize",
+ "get_frame_indices",
+ "get_padding_len",
+ "single_clip_reference_video_len",
+ "zigzag_padding",
+ "make_conditioning_mask",
+ "read_reference_video_frames",
+ "write_reference_video",
+]
+
+
+class LetterboxInfo(msgspec.Struct, frozen=True):
+ """Where letterbox_resize put the content inside the padded canvas."""
+
+ # Axis the padding bars sit on: "width" pads left/right, "height" pads top/bottom.
+ pad_axis: Literal["width", "height"]
+ # Bar size in pixels before the content along pad_axis.
+ padding: int
+ # Content extent in pixels along pad_axis.
+ content_len: int
+
+ def crop(self, frames: np.ndarray) -> np.ndarray:
+ """Cut the bars back out of ``[T, H, W, C]`` frames."""
+ start, stop = self.padding, self.padding + self.content_len
+ if self.pad_axis == "width":
+ return frames[:, :, start:stop, :]
+ return frames[:, start:stop, :, :]
+
+
+def resize_by_area(
+ image: np.ndarray,
+ target_area: int,
+ keep_aspect_ratio: bool = True,
+ divisor: int = 16,
+ padding_color: tuple[int, int, int] = (0, 0, 0),
+) -> tuple[np.ndarray, LetterboxInfo]:
+ """Resize ``image`` (``[H, W, C]`` uint8) so its area is <= ``target_area``, then letterbox-pad to that size."""
+ h, w = image.shape[:2]
+ aspect_ratio = w / h
+ if keep_aspect_ratio:
+ height_target = math.sqrt(target_area / aspect_ratio)
+ width_target = target_area / height_target
+ else:
+ width_target = height_target = math.sqrt(target_area)
+ width_target, height_target = (
+ int((width_target // divisor) * divisor),
+ int((height_target // divisor) * divisor),
+ )
+
+ # Resize (INTER_AREA when shrinking, INTER_LINEAR when enlarging)
+ # Taken as is from the reference implementation.
+ interpolation = (
+ cv2.INTER_AREA if (width_target * height_target < w * h) else cv2.INTER_LINEAR
+ )
+
+ return letterbox_resize(
+ image,
+ height=height_target,
+ width=width_target,
+ padding_color=padding_color,
+ interpolation=interpolation,
+ )
+
+
+def letterbox_resize(
+ image: np.ndarray,
+ height: int = 512,
+ width: int = 512,
+ padding_color: tuple[int, int, int] = (0, 0, 0),
+ interpolation: int = cv2.INTER_LINEAR,
+) -> tuple[np.ndarray, LetterboxInfo]:
+ """Aspect-preserving resize of ``image`` into a ``[height, width, C]`` uint8 canvas with letterbox padding."""
+ height_source = image.shape[0]
+ width_source = image.shape[1]
+ num_channels_source = image.shape[2]
+
+ canvas = np.zeros((height, width, num_channels_source))
+ if num_channels_source == 1:
+ canvas[:, :, 0] = padding_color[0]
+ else:
+ canvas[:, :, 0] = padding_color[0]
+ canvas[:, :, 1] = padding_color[1]
+ canvas[:, :, 2] = padding_color[2]
+
+ if (height_source / width_source) > (height / width):
+ width_target = int(height / height_source * width_source)
+ img = cv2.resize(image, (width_target, height), interpolation=interpolation)
+ padding = int((width - width_target) / 2)
+ if len(img.shape) == 2:
+ img = img[
+ :, :, np.newaxis
+ ] # grayscale image: add a num_channels_source dimension
+ canvas[:, padding : padding + width_target, :] = img
+ pad_axis = "width"
+ content_len = width_target
+ else:
+ height_target = int(width / width_source * height_source)
+ img = cv2.resize(image, (width, height_target), interpolation=interpolation)
+ padding = int((height - height_target) / 2)
+ if len(img.shape) == 2:
+ img = img[
+ :, :, np.newaxis
+ ] # grayscale image: add a num_channels_source dimension
+ canvas[padding : padding + height_target, :, :] = img
+ pad_axis = "height"
+ content_len = height_target
+
+ canvas = np.uint8(canvas)
+ return canvas, LetterboxInfo(
+ pad_axis=pad_axis, padding=padding, content_len=content_len
+ )
+
+
+def get_frame_indices(
+ num_frames_source: int, fps_source: float, num_frames_target: int, fps_target: float
+) -> list[int]:
+ """Source frame index (``list[int]``, nearest frame clipped to the source range) for each of the
+ ``num_frames_target`` output frames when resampling from ``fps_source`` to ``fps_target``."""
+
+ times = np.arange(0, num_frames_target) / fps_target
+ frame_indices_source = np.round(times * fps_source).astype(int)
+ frame_indices_source = np.clip(frame_indices_source, 0, num_frames_source - 1)
+
+ return frame_indices_source.tolist()
+
+
+# Fewest frames the last clip may add beyond its overlap; a multiple of 4 so it also sits
+# on the temporal-VAE grid. Taken as is from the reference.
+_MIN_LAST_CLIP_NEW_FRAMES = 28
+
+
+def get_padding_len(
+ num_frames: int, clip_len: int, num_frames_conditioning: int = 1
+) -> int:
+ """Reference-video length after zigzag padding so the last clip is long enough and
+ 4k-aligned; taken as is from the reference implementation."""
+
+ remaining = (num_frames - num_frames_conditioning) % (
+ clip_len - num_frames_conditioning
+ )
+ # Short leftover: raise it to the minimum. Otherwise: round up to the next multiple
+ # of 4 (VAE temporal stride); the minimum is itself a multiple of 4.
+ if remaining < _MIN_LAST_CLIP_NEW_FRAMES:
+ padding_needed = _MIN_LAST_CLIP_NEW_FRAMES - remaining
+ else:
+ padding_needed = 4 - remaining % 4
+
+ return num_frames + padding_needed
+
+
+# get_padding_len extends every reference video to at least this many frames, so a shorter
+# clip_len can never hold the whole video in one clip.
+MIN_SINGLE_CLIP_LEN = _MIN_LAST_CLIP_NEW_FRAMES + 1
+
+
+def single_clip_reference_video_len(
+ clip_len: int, num_frames_conditioning: int = 1
+) -> int:
+ """Reference-video frame count that get_padding_len extends to exactly ``clip_len``,
+ so build_schedule yields one full-length clip; used to size the synthetic warmup video."""
+ num_frames = clip_len - 1
+ if (
+ clip_len < MIN_SINGLE_CLIP_LEN
+ or get_padding_len(num_frames, clip_len, num_frames_conditioning) != clip_len
+ ):
+ raise ValueError(
+ f"clip_len={clip_len} cannot hold a reference video in a single clip; "
+ f"the schedule needs clip_len >= {MIN_SINGLE_CLIP_LEN}"
+ )
+ return num_frames
+
+
+def zigzag_padding(array: list, target_len: int) -> list:
+ """Pad ``array`` up to ``target_len`` by bouncing the read index back and forth (deep copies)."""
+ if target_len < len(array):
+ raise ValueError(
+ f"zigzag_padding only extends: target_len {target_len} < len(array) {len(array)}"
+ )
+ index = 0
+ flip = False
+ target_array = []
+ while len(target_array) < target_len:
+ target_array.append(deepcopy(array[index]))
+ if flip:
+ index -= 1
+ else:
+ index += 1
+ if index == 0 or index == len(array) - 1:
+ flip = not flip
+ return target_array
+
+
+def make_conditioning_mask(
+ latent_t: int,
+ latent_h: int,
+ latent_w: int,
+ num_prefix_conditioning_frames: int = 1,
+ device: torch.device | str = "cuda",
+) -> torch.Tensor:
+ """Build the ``[4, latent_t, latent_h, latent_w]`` fp32 i2v conditioning mask; ``num_prefix_conditioning_frames`` is the
+ conditioned prefix length in pixel frames."""
+
+ # The VAE maps the first frame to 1 frame in latent space.
+ # For each subsequent 4 pixel frames it maps them to 1 frame in the latent space.
+ # Hence, the ``(latent_t - 1) * 4 + 1``.
+ # VAE: 4k + 1 pixel frames -> k + 1 latent frames
+ mask = torch.zeros(1, (latent_t - 1) * 4 + 1, latent_h, latent_w, device=device)
+
+ mask[:, :num_prefix_conditioning_frames] = 1
+ mask = torch.concat(
+ [torch.repeat_interleave(mask[:, 0:1], repeats=4, dim=1), mask[:, 1:]], dim=1
+ ) # repeat the first pixel frame's flag 4x: (k - 1) * 4 + 1 -> 4 * k pixel slots
+
+ mask = mask.view(1, mask.shape[1] // 4, 4, latent_h, latent_w)
+ # (1, 4*(k + 1), latent_h, latent_w) -> (1, (k + 1), 4, latent_h, latent_w)
+
+ mask = mask.transpose(1, 2)[0]
+ # (1, (k + 1), 4, latent_h, latent_w) -> (4, (k + 1), latent_h, latent_w)
+ return mask
+
+
+def read_reference_video_frames(path: str, fps_target: int) -> np.ndarray:
+ """Decode the reference video resampled to ``fps_target``; returns ``[N, H, W, 3]`` uint8 RGB.
+
+ Three readers: decord on CPU as the reference implementation does (its GPU decoder
+ exists only in CUDA builds); decord2 is the arm64 build of the same API; PyAV is the
+ fallback when neither is installed.
+ """
+ source = (
+ BytesIO(get_video_bytes(path))
+ if path.startswith(("http://", "https://", "data:"))
+ else path
+ )
+ try:
+ from decord import VideoReader
+ except ImportError:
+ try:
+ from decord2 import VideoReader
+ except ImportError:
+ VideoReader = None
+
+ if VideoReader is not None:
+ video_reader = VideoReader(source)
+ num_frames_source = len(video_reader)
+ fps_source = video_reader.get_avg_fps()
+ num_frames_target = int(num_frames_source / fps_source * fps_target)
+ indices = get_frame_indices(
+ num_frames_source, fps_source, num_frames_target, fps_target
+ )
+ return video_reader.get_batch(indices).asnumpy()
+
+ import av
+
+ # Decode everything once so num_frames_source matches decord's len() exactly.
+ with av.open(source) as container:
+ stream = container.streams.video[0]
+ fps_source = float(stream.average_rate)
+ frames_source = [
+ frame.to_ndarray(format="rgb24") for frame in container.decode(video=0)
+ ]
+ num_frames_source = len(frames_source)
+ num_frames_target = int(num_frames_source / fps_source * fps_target)
+ indices = get_frame_indices(
+ num_frames_source, fps_source, num_frames_target, fps_target
+ )
+ return np.stack([frames_source[i] for i in indices])
+
+
+def write_reference_video(path: str, frames: np.ndarray, fps: int) -> None:
+ """Write ``[N, H, W, 3]`` uint8 RGB frames to ``path`` as an mp4 at ``fps`` through the
+ runtime's video writer (the one request outputs go through), so the synthetic warmup
+ video is encoded exactly like a request output and decodes with the readers above."""
+ from sglang.multimodal_gen.configs.sample.sampling_params import DataType
+ from sglang.multimodal_gen.runtime.entrypoints.utils import post_process_sample
+
+ try:
+ post_process_sample(frames, DataType.VIDEO, fps, save_file_path=path)
+ except Exception as e:
+ raise RuntimeError(
+ f"the shared video writer (post_process_sample) could not write the "
+ f"synthetic reference video {path}: {e}"
+ ) from e
+
+
+def validate_and_get_single_string(value: str | list[str] | None, name: str) -> str:
+ """This pipeline produces one video per request, so a batched prompt field,
+ video path or image path must hold exactly one string."""
+ if isinstance(value, list):
+ if len(value) != 1:
+ raise ValueError(
+ f"Wan-Animate-2 takes exactly one '{name}' per request, got {len(value)}"
+ )
+ value = value[0]
+ if not isinstance(value, str):
+ raise ValueError(
+ f"Wan-Animate-2 requires '{name}' to be a string, got {value!r}"
+ )
+ return value
diff --git a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
index 270ef2b809..53c307eb48 100644
--- a/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
+++ b/python/sglang/multimodal_gen/test/server/consistency_thresholds/h100.json
@@ -270,6 +270,18 @@
"ssim_threshold": 0.48,
"psnr_threshold": 13.0,
"mean_abs_diff_threshold": 45.0
+ },
+ "wan2_2_animate_2_14b_ref2v": {
+ "clip_threshold": 0.90,
+ "ssim_threshold": 0.92,
+ "psnr_threshold": 24.0,
+ "mean_abs_diff_threshold": 10.0
+ },
+ "wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
+ "clip_threshold": 0.90,
+ "ssim_threshold": 0.92,
+ "psnr_threshold": 24.0,
+ "mean_abs_diff_threshold": 10.0
}
},
"default_clip_threshold_image": 0.92,
diff --git a/python/sglang/multimodal_gen/test/server/gpu_cases.py b/python/sglang/multimodal_gen/test/server/gpu_cases.py
index b0b82a5534..d4385f8861 100644
--- a/python/sglang/multimodal_gen/test/server/gpu_cases.py
+++ b/python/sglang/multimodal_gen/test/server/gpu_cases.py
@@ -39,6 +39,7 @@ from sglang.multimodal_gen.test.server.testcase_configs import (
SANA_WM_TI2V_CI_sampling_params,
T2I_sampling_params,
T2V_sampling_params,
+ WAN_2_2_ANIMATE_2_14B_sampling_params,
_make_modelopt_ci_case,
_with_default_num_gpus,
)
@@ -63,6 +64,7 @@ from sglang.multimodal_gen.test.test_utils import (
DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_1_T2V_14B_MODEL_NAME_FOR_TEST,
+ DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST,
DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST,
@@ -459,6 +461,23 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
run_models_api_check=False,
run_t2v_input_reference_check=False,
),
+ # Reference image + driving video -> video. The DiT peaks at 76 GB resident,
+ # so the 80 GB lane streams it layerwise; this also covers the uncond block-9 skip.
+ DiffusionTestCase(
+ "wan2_2_animate_2_14b_ref2v",
+ DiffusionServerArgs(
+ model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
+ modality="video",
+ dit_layerwise_offload=True,
+ text_encoder_cpu_offload=True,
+ extras=["--layerwise-offload-components", "transformer"],
+ ),
+ WAN_2_2_ANIMATE_2_14B_sampling_params,
+ # The DiT forward takes a per-clip conditioning container and runs forward_ref,
+ # so the raw-component harness contract does not apply.
+ run_perf_check=True,
+ run_component_accuracy_check=False,
+ ),
# flaky
# === Helios T2V ===
# DiffusionTestCase(
@@ -975,6 +994,32 @@ TWO_GPU_CASES = [
model_path=DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST,
),
),
+ # GT = the 1-GPU frames renamed. With replicated encoders, serial VAE decode and the
+ # VAE channels_last_3d layout the loader otherwise applies only for num_gpus == 1,
+ # CFG-parallel output is bit-identical to the 1-GPU case, so both share one ground truth.
+ DiffusionTestCase(
+ "wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu",
+ DiffusionServerArgs(
+ model_path=DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST,
+ modality="video",
+ cfg_parallel=True,
+ # On 80 GB cards the resident DiT leaves no room for the full-shape warmup probe; stream it layerwise (bit-identical output).
+ dit_layerwise_offload=True,
+ text_encoder_cpu_offload=True,
+ extras=[
+ "--layerwise-offload-components",
+ "transformer",
+ "--encoder-parallel",
+ "replicate",
+ "--vae-config.use-parallel-decode",
+ "false",
+ ],
+ env_vars={"SGLANG_DIFFUSION_VAE_CHANNELS_LAST_3D": "1"},
+ ),
+ WAN_2_2_ANIMATE_2_14B_sampling_params,
+ run_perf_check=True,
+ run_component_accuracy_check=False,
+ ),
DiffusionTestCase(
"wan2_2_t2v_a14b_2gpu",
DiffusionServerArgs(
diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
index 0eb17fca3a..8b8040d023 100644
--- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
+++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json
@@ -3524,6 +3524,60 @@
"expected_avg_denoise_ms": 0.0,
"expected_median_denoise_ms": 0.0,
"estimated_full_test_time_s": 73.7
+ },
+ "wan2_2_animate_2_14b_ref2v": {
+ "stages_ms": {
+ "WanAnimate2BeforeDenoisingStage": 3783.78,
+ "WanAnimate2DenoisingStage": 94604.96,
+ "WanAnimate2OutputStage": 0.06
+ },
+ "denoise_step_ms": {
+ "0": 25832.83,
+ "1": 7210.15,
+ "2": 11187.28,
+ "3": 8460.7,
+ "4": 8407.93,
+ "5": 8409.07,
+ "6": 8408.07,
+ "7": 8415.78
+ },
+ "expected_e2e_ms": 98402.77,
+ "expected_load_ms": 62901.15,
+ "expected_avg_denoise_ms": 10791.48,
+ "expected_median_denoise_ms": 8412.42,
+ "load_peak_vram_mb": 1776.0,
+ "runtime_peak_vram_mb": 36322.0,
+ "load_peak_allocated_mb": 1737.0,
+ "runtime_peak_allocated_mb": 25748.0,
+ "estimated_full_test_time_s": 489.9,
+ "warmup_peak_vram_mb": 14598.0
+ },
+ "wan2_2_animate_2_14b_ref2v_cfg_parallel_2gpu": {
+ "stages_ms": {
+ "WanAnimate2BeforeDenoisingStage": 5778.62,
+ "WanAnimate2DenoisingStage": 58196.97,
+ "WanAnimate2OutputStage": 0.06
+ },
+ "denoise_step_ms": {
+ "0": 21723.06,
+ "1": 3125.83,
+ "2": 4335.41,
+ "3": 4337.63,
+ "4": 4347.53,
+ "5": 4339.76,
+ "6": 4350.68,
+ "7": 4280.35
+ },
+ "expected_e2e_ms": 63988.59,
+ "expected_load_ms": 64309.0,
+ "expected_avg_denoise_ms": 6355.03,
+ "expected_median_denoise_ms": 4338.7,
+ "load_peak_vram_mb": 1776.0,
+ "runtime_peak_vram_mb": 36322.0,
+ "load_peak_allocated_mb": 1737.0,
+ "runtime_peak_allocated_mb": 25745.0,
+ "estimated_full_test_time_s": 404.0,
+ "warmup_peak_vram_mb": 14598.0
}
}
}
diff --git a/python/sglang/multimodal_gen/test/server/testcase_configs.py b/python/sglang/multimodal_gen/test/server/testcase_configs.py
index 19b811a40c..7bb78d0258 100644
--- a/python/sglang/multimodal_gen/test/server/testcase_configs.py
+++ b/python/sglang/multimodal_gen/test/server/testcase_configs.py
@@ -36,6 +36,10 @@ from sglang.multimodal_gen.registry import (
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.perf_logger import RequestPerfRecord
+from sglang.multimodal_gen.test.test_utils import (
+ SGL_TEST_FILES_CI_DATA_REPO,
+ SGL_TEST_FILES_CI_DATA_REVISION,
+)
@dataclass
@@ -885,6 +889,35 @@ TURBOWAN_I2V_sampling_params = DiffusionSamplingParams(
fps=4,
)
+# Wan-Animate-2 inputs; they ship in the same ci-data-diffusion commit as the GT,
+# so the pinned revision must point at a commit that carries them.
+_WAN_2_2_ANIMATE_2_CI_INPUT_BASE = (
+ "https://raw.githubusercontent.com/"
+ f"{SGL_TEST_FILES_CI_DATA_REPO}/{SGL_TEST_FILES_CI_DATA_REVISION}/"
+ "diffusion-ci/inputs/wan_animate_2"
+)
+WAN_2_2_ANIMATE_2_14B_sampling_params = DiffusionSamplingParams(
+ prompt=(
+ "Character appearance description: a smiling young woman with shoulder-length "
+ "dark hair, a mustard-yellow knit sweater, blue jeans and white sneakers. "
+ "Background description: a plain light-gray studio backdrop."
+ ),
+ image_path=f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/reference_short.jpg",
+ # The model reads `size` as an area budget and emits the 16-aligned canvas fitted to
+ # the reference aspect; 576x1024 (9:16, like the image) maps onto itself.
+ output_size="576x1024",
+ fps=16,
+ output_format="mp4",
+ # 24 driving frames at 16 fps pad to one 29-frame clip (get_padding_len),
+ # so the request denoises a single clip; output length follows the driving video.
+ extras={
+ "video_url": f"{_WAN_2_2_ANIMATE_2_CI_INPUT_BASE}/driving_video_short.mp4",
+ "clip_len": 37,
+ "num_inference_steps": 8,
+ "seed": 42,
+ },
+)
+
HUNYUAN3D_SHAPE_sampling_params = DiffusionSamplingParams(
prompt="",
image_path="https://raw.githubusercontent.com/sgl-project/sgl-test-files/main/diffusion-ci/consistency_gt/1-gpu/hunyuan3d_2_0/hunyuan3d.png",
diff --git a/python/sglang/multimodal_gen/test/test_utils.py b/python/sglang/multimodal_gen/test/test_utils.py
index df0e4ba14b..6b7cde1725 100644
--- a/python/sglang/multimodal_gen/test/test_utils.py
+++ b/python/sglang/multimodal_gen/test/test_utils.py
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
# publish.
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
-SGL_TEST_FILES_CI_DATA_REVISION = "17312c8df37e65bffe75818884bc2a5c93cf3094"
+SGL_TEST_FILES_CI_DATA_REVISION = "31d8ad69f2434727aad518f0678fbd57823856fe"
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
# when it's regenerated on its own cadence.
@@ -200,6 +200,9 @@ DEFAULT_WAN_2_1_I2V_14B_720P_MODEL_NAME_FOR_TEST = (
DEFAULT_WAN_2_2_TI2V_5B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
DEFAULT_WAN_2_2_T2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
DEFAULT_WAN_2_2_I2V_A14B_MODEL_NAME_FOR_TEST = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
+DEFAULT_WAN_2_2_ANIMATE_2_14B_MODEL_NAME_FOR_TEST = (
+ "Wan-AI/Wan2.2-Animate-2-14B-Diffusers"
+)
# MOVA video generation models
DEFAULT_MOVA_360P_MODEL_NAME_FOR_TEST = "OpenMOSS-Team/MOVA-360p"
diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py
new file mode 100644
index 0000000000..8cb3af6d0b
--- /dev/null
+++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_config.py
@@ -0,0 +1,1763 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Unit tests for the Wan-Animate-2 configs, registry wiring, request handling and the
+preprocessing helpers that feed the conditioning tensors.
+
+No models, GPU, or network: the registry detector tests patch
+``maybe_download_model_index``; stages run with stand-in encoders.
+"""
+
+import logging
+import math
+import os
+from dataclasses import fields, replace
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import numpy as np
+import pytest
+import torch
+from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
+from transformers import BatchEncoding
+
+from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import (
+ WanAnimate2ArchConfig,
+ WanAnimate2Config,
+)
+from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan_Animate_2_14B_Config
+from sglang.multimodal_gen.configs.sample.sampling_params import (
+ DataType,
+ SamplingParams,
+)
+from sglang.multimodal_gen.configs.sample.wan import Wan_Animate_2_14B_SamplingParam
+from sglang.multimodal_gen.registry import _get_config_info
+from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
+from sglang.multimodal_gen.runtime.distributed.parallel_state import (
+ maybe_init_distributed_environment_and_model_parallel,
+ model_parallel_is_initialized,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
+ VideoGenerationsRequest,
+)
+from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
+ get_sampling_request_extra_fields,
+)
+from sglang.multimodal_gen.runtime.layers.attention.backends.attention_backend import (
+ AttentionBackendEnum,
+)
+from sglang.multimodal_gen.runtime.layers.linear import UnquantizedLinearMethod
+from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
+ _supported_init_kwargs,
+)
+from sglang.multimodal_gen.runtime.loader.component_loaders.transformer_loader import (
+ TransformerLoader,
+)
+from sglang.multimodal_gen.runtime.loader.utils import (
+ component_name_to_loader_cls,
+ get_param_names_mapping,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import (
+ WanAnimate2Transformer3DModel,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
+ WanAnimate2TransformerBlock,
+)
+from sglang.multimodal_gen.runtime.models.schedulers.scheduling_dpm_solver_multistep import (
+ DPMSolverMultistepScheduler,
+)
+from sglang.multimodal_gen.runtime.pipelines.wan_animate_2_pipeline import (
+ WanAnimate2Pipeline,
+ WanAnimate2TransformerLoader,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
+from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising import (
+ WanAnimate2BeforeDenoisingStage,
+ WanAnimate2RequestState,
+ _WanAnimate2Inputs,
+ build_clip_conditioning,
+ build_schedule,
+ generation_token_grid,
+ request_state_from_batch,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
+ WanAnimate2DenoisingStage,
+ get_sampling_sigmas,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.encoder_adapters import (
+ WanAnimate2VaeAdapter,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.preprocess import (
+ LetterboxInfo,
+ get_frame_indices,
+ get_padding_len,
+ letterbox_resize,
+ make_conditioning_mask,
+ read_reference_video_frames,
+ resize_by_area,
+ write_reference_video,
+ zigzag_padding,
+)
+from sglang.multimodal_gen.runtime.server_args import ServerArgs, set_global_server_args
+from sglang.multimodal_gen.runtime.server_warmup import prepare_warmup_image_path
+from sglang.multimodal_gen.runtime.warmup_request_builder import (
+ SERVER_WARMUP_VIDEO_STEPS,
+ build_warmup_reqs,
+ lighten_warmup_req,
+)
+from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
+ ensure_distributed_env_defaults,
+)
+
+# Wan_Animate_2_14B_Config
+
+
+def test_pipeline_local_loader_preserves_shared_transformer_registry():
+ for name in TransformerLoader.component_names:
+ assert component_name_to_loader_cls[name] is TransformerLoader
+ assert not TransformerLoader.strict_checkpoint_keys
+ assert WanAnimate2TransformerLoader.strict_checkpoint_keys
+ assert WanAnimate2Pipeline.component_loaders == {
+ "transformer": WanAnimate2TransformerLoader
+ }
+
+
+def test_text_encoder_counts_tokens_before_device_transfer():
+ config = Wan_Animate_2_14B_Config()
+ tokens = BatchEncoding(
+ {
+ "input_ids": torch.tensor([[1, 2, 0, 0]]),
+ "attention_mask": torch.tensor([[1, 1, 0, 0]]),
+ }
+ )
+ inputs = config.tokenize_prompt(["prompt"], lambda *args, **kwargs: tokens, {})
+ inputs.to("meta")
+ assert (
+ inputs["input_ids"].device.type
+ == inputs["attention_mask"].device.type
+ == "meta"
+ )
+ assert inputs["num_tokens"] == 2
+ output = SimpleNamespace(last_hidden_state=torch.empty(1, 4, 8, device="meta"))
+ assert config.postprocess_text_funcs[0](output, inputs).shape == (2, 8)
+ assert not WanAnimate2BeforeDenoisingStage.has_deduplicated_output_fields()
+
+
+def test_pipeline_keeps_native_modules_visible_to_memory_managers():
+ path = "sglang.multimodal_gen.runtime.pipelines.wan_animate_2_pipeline"
+ vae = torch.nn.Identity()
+ vae.latents_mean, vae.latents_std = [0.0], [1.0]
+ modules = {
+ "text_encoder": torch.nn.Identity(),
+ "image_encoder": torch.nn.Identity(),
+ "vae": vae,
+ "tokenizer": object(),
+ "image_processor": SimpleNamespace(
+ crop_size={"height": 224, "width": 224},
+ image_mean=(0.5, 0.5, 0.5),
+ image_std=(0.5, 0.5, 0.5),
+ ),
+ "transformer": object(),
+ "scheduler": object(),
+ }
+ stub = SimpleNamespace(
+ get_module=modules.__getitem__,
+ add_stage=lambda **kwargs: None,
+ add_stage_factory=lambda *args: None,
+ )
+ with (
+ patch(f"{path}.get_local_torch_device", return_value=torch.device("cpu")),
+ patch(f"{path}.WanAnimate2BeforeDenoisingStage") as before,
+ patch(f"{path}.WanAnimate2DenoisingStage") as denoise,
+ ):
+ WanAnimate2Pipeline.create_pipeline_stages(
+ stub, SimpleNamespace(pipeline_config=Wan_Animate_2_14B_Config())
+ )
+ args = before.call_args.kwargs
+ assert "load_modules" not in vars(WanAnimate2Pipeline)
+ assert args["text_encoder"] is modules["text_encoder"]
+ assert args["tokenizer"] is modules["tokenizer"]
+ assert args["image_encoder"].model is modules["image_encoder"]
+ assert args["vae"].vae is vae
+ assert denoise.call_args.kwargs["vae"] is args["vae"]
+
+
+def test_single_expert_has_no_boundary_switching():
+ # Single-expert model: no low-noise expert switch via boundary_ratio.
+ config = Wan_Animate_2_14B_Config()
+ assert config.boundary_ratio is None
+ assert config.dit_config.arch_config.boundary_ratio is None
+
+
+@pytest.mark.parametrize(
+ "tp_size, ulysses_degree, enable_cfg_parallel",
+ [
+ (2, 2, False), # TP2 x Ulysses2 (4 GPUs)
+ (2, 2, True), # TP2 x Ulysses2 auto-derived from --num-gpus 8 with CFG
+ (2, 1, False), # TP2 alone
+ (1, 2, False), # Ulysses2 alone
+ (4, 1, False), # TP4
+ (1, 2, True), # CFG-parallel x Ulysses2
+ (2, 1, True), # CFG-parallel x TP2
+ ],
+)
+def test_validate_server_args_accepts_tp_combined_with_ulysses(
+ tp_size, ulysses_degree, enable_cfg_parallel
+):
+ """TP, Ulysses and CFG-parallel compose freely as long as the degrees divide the
+ 40 attention heads; the hybrid TP x Ulysses layout starts like the single-axis ones."""
+ server_args = _parallel_server_args(
+ tp_size=tp_size,
+ ulysses_degree=ulysses_degree,
+ enable_cfg_parallel=enable_cfg_parallel,
+ )
+ Wan_Animate_2_14B_Config().validate_server_args(server_args)
+
+
+def _parallel_server_args(
+ *,
+ tp_size: int = 1,
+ ulysses_degree: int = 1,
+ ring_degree: int = 1,
+ enable_cfg_parallel: bool = False,
+ use_fsdp_inference: bool = False,
+ attention_backend: str | None = None,
+ attention_backend_explicit: bool = False,
+ component_attention_backends: dict[str, str] | None = None,
+) -> SimpleNamespace:
+ """The normalized ServerArgs fields validate_server_args reads (sp_degree = ulysses * ring)."""
+ requested = dict(component_attention_backends or {})
+ explicit = {"attention_backend"} if attention_backend_explicit else set()
+ return SimpleNamespace(
+ tp_size=tp_size,
+ sp_degree=ulysses_degree * ring_degree,
+ ulysses_degree=ulysses_degree,
+ ring_degree=ring_degree,
+ enable_cfg_parallel=enable_cfg_parallel,
+ use_fsdp_inference=use_fsdp_inference,
+ attention_backend=attention_backend,
+ is_arg_explicitly_set=lambda name: name in explicit,
+ requested_component_attention_backend=lambda name: requested.get(name),
+ )
+
+
+def test_validate_server_args_rejects_fsdp_before_any_weights_load():
+ with pytest.raises(NotImplementedError, match="use-fsdp-inference"):
+ Wan_Animate_2_14B_Config().validate_server_args(
+ _parallel_server_args(use_fsdp_inference=True)
+ )
+
+
+def test_validate_server_args_rejects_ring_sequence_parallelism():
+ with pytest.raises(NotImplementedError, match="ring sequence parallelism"):
+ Wan_Animate_2_14B_Config().validate_server_args(
+ _parallel_server_args(ulysses_degree=1, ring_degree=2)
+ )
+
+
+@pytest.mark.parametrize(
+ "tp_size, ulysses_degree, rejected",
+ [
+ (1, 8, False), # 40 heads / 8
+ (1, 3, True), # 40 heads not divisible by 3
+ (1, 16, True), # 40 heads not divisible by 16
+ (3, 1, True), # 40 heads not divisible by tp 3
+ (8, 1, False),
+ (2, 4, False), # 40 heads / tp 2 = 20 local heads, divisible by 4
+ (4, 2, False), # 40 heads / tp 4 = 10 local heads, divisible by 2
+ (4, 3, True), # 10 local heads not divisible by 3
+ (8, 4, True), # 5 local heads not divisible by 4
+ ],
+)
+def test_validate_server_args_checks_head_divisibility(
+ tp_size, ulysses_degree, rejected
+):
+ """Ulysses splits the TP-local heads across ranks; a non-dividing degree must be
+ rejected at startup, before any weights load."""
+ server_args = _parallel_server_args(tp_size=tp_size, ulysses_degree=ulysses_degree)
+ config = Wan_Animate_2_14B_Config()
+ if rejected:
+ with pytest.raises(ValueError, match="divisible by"):
+ config.validate_server_args(server_args)
+ else:
+ config.validate_server_args(server_args)
+
+
+@pytest.mark.parametrize(
+ "kwargs",
+ [
+ {"attention_backend": "fa", "attention_backend_explicit": True},
+ {"component_attention_backends": {"transformer": "torch_sdpa"}},
+ ],
+)
+def test_validate_server_args_rejects_a_requested_dit_attention_backend(kwargs):
+ # The in-context self-attention is flex_attention only, so a requested backend
+ # would be honoured by the cross-attention and silently dropped by the main path.
+ with pytest.raises(ValueError, match="flex_attention only"):
+ Wan_Animate_2_14B_Config().validate_server_args(_parallel_server_args(**kwargs))
+
+
+def test_validate_server_args_keeps_a_platform_default_attention_backend():
+ # ROCm fills attention_backend in when the user gave none; that is not a request.
+ Wan_Animate_2_14B_Config().validate_server_args(
+ _parallel_server_args(
+ attention_backend="aiter", attention_backend_explicit=False
+ )
+ )
+
+
+def test_validate_server_args_rejects_mxfp8_offline_qk_rotation(monkeypatch):
+ monkeypatch.setenv("SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION", "true")
+ with pytest.raises(ValueError, match="MXFP8"):
+ Wan_Animate_2_14B_Config().validate_server_args(_parallel_server_args())
+
+
+class _RotatedQuantConfig:
+ """A quantized checkpoint that ships offline Q/K rotation matrices for one block."""
+
+ def __init__(self, prefix: str) -> None:
+ self.quant_description = {
+ f"{prefix}.attn1.q_rot": "FLOAT",
+ f"{prefix}.attn1.k_rot": "FLOAT",
+ }
+
+ def get_quant_method(self, layer, prefix: str) -> UnquantizedLinearMethod:
+ return UnquantizedLinearMethod()
+
+
+def test_block_refuses_a_checkpoint_with_offline_qk_rotation(monkeypatch):
+ # The parent block registers q_rot / k_rot and applies them in its own attention
+ # path; forward_ref / forward_gen never do, so loading would be silently wrong.
+ monkeypatch.setenv("SGLANG_DIFFUSION_ENABLE_MXFP8_ATTENTION", "true")
+ _ensure_single_process_parallel_runtime()
+ with torch.device("meta"), pytest.raises(ValueError, match="offline Q/K rotation"):
+ WanAnimate2TransformerBlock(
+ 64,
+ 128,
+ 2,
+ "rms_norm",
+ True,
+ 1e-6,
+ None,
+ {AttentionBackendEnum.TORCH_SDPA},
+ prefix="blocks.0",
+ quant_config=_RotatedQuantConfig("blocks.0"),
+ )
+
+
+# WanAnimate2ArchConfig / WanAnimate2Config
+
+
+def test_in_out_channels_match_in_context_io():
+ # in_channels = 16 noise + 20 conditioning; out_channels = 16 latent.
+ arch = WanAnimate2ArchConfig()
+ assert arch.in_channels == 36
+ assert arch.out_channels == 16
+
+
+def test_backbone_dims_reuse_wan22_i2v_14b():
+ arch = WanAnimate2ArchConfig()
+ assert arch.num_layers == 40
+ assert arch.image_dim == 1280
+ assert arch.added_kv_proj_dim == 5120
+
+
+def test_post_init_derives_hidden_size_and_latent_channels():
+ arch = WanAnimate2ArchConfig()
+ assert arch.num_attention_heads == 40
+ assert arch.attention_head_dim == 128
+ assert arch.hidden_size == 5120
+ assert arch.num_channels_latents == 16
+
+
+# The 1303 tensor names of Wan2.2-Animate-2-14B-Diffusers/transformer (the safetensors
+# index's weight_map): 23 top-level tensors plus 32 per block for 40 blocks.
+_CHECKPOINT_TOP_LEVEL_KEYS = (
+ "head.head.bias",
+ "head.head.weight",
+ "head.modulation",
+ "img_emb.proj.0.bias",
+ "img_emb.proj.0.weight",
+ "img_emb.proj.1.bias",
+ "img_emb.proj.1.weight",
+ "img_emb.proj.3.bias",
+ "img_emb.proj.3.weight",
+ "img_emb.proj.4.bias",
+ "img_emb.proj.4.weight",
+ "patch_embedding.bias",
+ "patch_embedding.weight",
+ "text_embedding.0.bias",
+ "text_embedding.0.weight",
+ "text_embedding.2.bias",
+ "text_embedding.2.weight",
+ "time_embedding.0.bias",
+ "time_embedding.0.weight",
+ "time_embedding.2.bias",
+ "time_embedding.2.weight",
+ "time_projection.1.bias",
+ "time_projection.1.weight",
+)
+_CHECKPOINT_BLOCK_KEYS = (
+ "cross_attn.add_k_proj.bias",
+ "cross_attn.add_k_proj.weight",
+ "cross_attn.add_v_proj.bias",
+ "cross_attn.add_v_proj.weight",
+ "cross_attn.norm_added_k.weight",
+ "cross_attn.norm_k.weight",
+ "cross_attn.norm_q.weight",
+ "cross_attn.to_k.bias",
+ "cross_attn.to_k.weight",
+ "cross_attn.to_out.0.bias",
+ "cross_attn.to_out.0.weight",
+ "cross_attn.to_q.bias",
+ "cross_attn.to_q.weight",
+ "cross_attn.to_v.bias",
+ "cross_attn.to_v.weight",
+ "ffn.0.bias",
+ "ffn.0.weight",
+ "ffn.2.bias",
+ "ffn.2.weight",
+ "modulation",
+ "norm3.bias",
+ "norm3.weight",
+ "self_attn.norm_k.weight",
+ "self_attn.norm_q.weight",
+ "self_attn.to_k.bias",
+ "self_attn.to_k.weight",
+ "self_attn.to_out.0.bias",
+ "self_attn.to_out.0.weight",
+ "self_attn.to_q.bias",
+ "self_attn.to_q.weight",
+ "self_attn.to_v.bias",
+ "self_attn.to_v.weight",
+)
+
+
+def _checkpoint_keys() -> list[str]:
+ keys = list(_CHECKPOINT_TOP_LEVEL_KEYS)
+ for block in range(WanAnimate2ArchConfig().num_layers):
+ keys.extend(f"blocks.{block}.{key}" for key in _CHECKPOINT_BLOCK_KEYS)
+ return keys
+
+
+def test_every_checkpoint_key_maps_onto_a_distinct_dit_parameter():
+ """The DiT loads with the shared Wan rules plus the Wan-Animate-2 extras; a rule that
+ stops matching leaves a tensor unmapped (skipped with a warning) or lands two tensors on
+ one parameter, both of which load silently wrong weights."""
+ keys = _checkpoint_keys()
+ assert len(keys) == 1303
+ map_fn = get_param_names_mapping(WanAnimate2ArchConfig().param_names_mapping)
+ mapped = {key: map_fn(key)[0] for key in keys}
+ unmapped = sorted(key for key, name in mapped.items() if name == key)
+ assert unmapped == []
+ assert len(set(mapped.values())) == len(keys)
+
+ _ensure_single_process_parallel_runtime()
+ with torch.device("meta"):
+ model = WanAnimate2Transformer3DModel(WanAnimate2Config(), hf_config={})
+ assert set(mapped.values()) == set(model.state_dict())
+
+
+def _ensure_single_process_parallel_runtime() -> None:
+ # The DiT's parallel linear layers need the TP group even on a meta device.
+ if model_parallel_is_initialized():
+ return
+ ensure_distributed_env_defaults()
+ maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
+
+
+def test_official_checkpoint_config_names_update_the_arch_config():
+ """transformer/config.json carries the official names (dim, in_dim, num_heads, ...);
+ the loader's update_model_arch must land them on the fields the model reads."""
+ config = WanAnimate2Config()
+ config.update_model_arch(
+ {
+ "dim": 5120,
+ "in_dim": 36,
+ "out_dim": 16,
+ "num_heads": 40,
+ "patch_size": [1, 2, 2],
+ "refer_offset_t": 2,
+ "log_scale": -1.3,
+ "text_len": 512,
+ }
+ )
+ assert config.num_attention_heads == 40
+ assert config.attention_head_dim == 128
+ assert config.hidden_size == 5120
+ assert config.in_channels == 36
+ assert config.out_channels == 16
+ assert config.patch_size == (1, 2, 2)
+ assert config.refer_offset_t == 2
+ assert config.log_scale == -1.3
+ assert config.text_len == 512
+
+
+def test_text_len_defaults_to_the_checkpoint_value():
+ """The pipeline pads prompts to ``arch_config.text_len``: the checkpoint's
+ transformer/config.json sets it and the default covers a config without the key."""
+ assert WanAnimate2ArchConfig().text_len == 512
+ assert Wan_Animate_2_14B_Config().dit_config.arch_config.text_len == 512
+
+
+def test_checkpoint_config_without_image_embedding_is_rejected():
+ # The model always builds the image-embedding cross-attention; a checkpoint that
+ # declares use_img_emb false would load and run the wrong architecture silently.
+ config = WanAnimate2Config()
+ with pytest.raises(ValueError, match="use_img_emb"):
+ config.update_model_arch({"use_img_emb": False})
+
+
+# Wan_Animate_2_14B_SamplingParam
+
+
+@pytest.mark.parametrize(
+ "requested, expected",
+ [
+ (37, 37), # 4k+1 -> unchanged
+ (80, 81), # 4k -> up one frame
+ (42, 41), # 4k+2 -> down
+ (43, 41), # 4k+3 -> down
+ (4, 5), # smallest accepted clip
+ ],
+)
+def test_clip_len_rounds_to_4k_plus_1(requested, expected, caplog):
+ # VAE temporal stride is 4, so a clip must be 4k+1 frames; the rounding is logged
+ # once, and only when it changed the value.
+ with caplog.at_level(logging.INFO):
+ params = Wan_Animate_2_14B_SamplingParam(clip_len=requested)
+ assert params.clip_len == expected
+ assert params.clip_len % 4 == 1
+ rounding_logs = [r for r in caplog.records if "is not 4k+1" in r.getMessage()]
+ assert len(rounding_logs) == (0 if requested == expected else 1)
+ if rounding_logs:
+ assert f"clip_len={requested}" in rounding_logs[0].getMessage()
+ assert f"clip_len={expected}" in rounding_logs[0].getMessage()
+
+
+@pytest.mark.parametrize("requested", [-3, 0, 1, 2, 3])
+def test_clip_len_below_one_temporal_stride_is_rejected(requested):
+ # A clip shorter than one VAE temporal stride is rejected at admission, not in a stage.
+ with pytest.raises(ValueError, match=rf"clip_len={requested} is not supported"):
+ Wan_Animate_2_14B_SamplingParam(clip_len=requested)
+
+
+def test_num_frames_is_not_mirrored_from_clip_len():
+ """The output length follows the reference video, so clip_len must not be written
+ into num_frames: a mirrored value made `dataclasses.replace(num_frames=...)` snap back,
+ and the auto-residency probe (which shrinks a request by replacing num_frames until its
+ estimate fits) looped forever on cards where the full probe did not fit."""
+ params = Wan_Animate_2_14B_SamplingParam(clip_len=37)
+ assert params.num_frames == SamplingParams.num_frames
+ assert params.adjust_frames is False
+
+ lighter = replace(Wan_Animate_2_14B_SamplingParam(num_frames=37), num_frames=17)
+ assert lighter.num_frames == 17
+ assert lighter.clip_len == 37
+
+
+def test_lighten_warmup_req_shrinks_a_wan_animate_2_probe(tmp_path):
+ # Each shrink step must strictly reduce width*height*num_frames, the quantity the
+ # auto-residency probe loop terminates on; the ladder bottoms out at the 16-pixel floor.
+ server_args = _server_args(str(tmp_path))
+ req = Req(
+ sampling_params=Wan_Animate_2_14B_SamplingParam(num_frames=37),
+ prompt="probe",
+ width=640,
+ height=800,
+ )
+ units = []
+ for _ in range(64):
+ params = req.sampling_params
+ units.append(params.width * params.height * params.num_frames)
+ req = lighten_warmup_req(server_args, req)
+ if req is None:
+ break
+ assert req is None, "the shrink ladder never bottomed out"
+ assert units == sorted(units, reverse=True) and len(set(units)) == len(units)
+
+
+def test_multi_gpu_frame_alignment_leaves_num_frames_alone(tmp_path, caplog):
+ # The stages read clip_len, so the multi-GPU frame alignment must not touch num_frames.
+ server_args = _server_args(str(tmp_path))
+ server_args.num_gpus = 4
+ server_args.comfyui_mode = True
+ params = Wan_Animate_2_14B_SamplingParam(
+ prompt="p", image_path="ref.png", video_path="drv.mp4", num_frames=37
+ )
+ with caplog.at_level(logging.INFO):
+ params._adjust_visual_fields(server_args, server_args.pipeline_config)
+ assert params.num_frames == 37
+ messages = [r.getMessage() for r in caplog.records]
+ assert not any("based on number of GPUs" in m for m in messages)
+ assert any("ignores num_frames=37" in m for m in messages)
+
+
+@pytest.mark.parametrize("field_name", ["enable_teacache", "enable_spectrum"])
+def test_sampling_params_reject_the_shared_dit_cache_heuristics(field_name):
+ # Every block runs at every step in the in-context loop; the flags would be accepted
+ # and change nothing.
+ with pytest.raises(ValueError, match="does not support"):
+ Wan_Animate_2_14B_SamplingParam(**{field_name: True})
+
+
+# Registry resolution (no network)
+
+
+def test_registered_hf_path_resolves_to_wan_animate_configs():
+ info = _get_config_info("Wan-AI/Wan2.2-Animate-2-14B-Diffusers")
+ assert info is not None
+ assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
+ assert info.sampling_param_cls is Wan_Animate_2_14B_SamplingParam
+
+
+@pytest.mark.parametrize(
+ "path", ["some/local/dir/wan_animate_2", "my-wan2.2-animate-2-ckpt"]
+)
+def test_detector_resolves_wan_animate_style_paths(path):
+ with patch(
+ "sglang.multimodal_gen.registry.maybe_download_model_index",
+ return_value={},
+ ):
+ info = _get_config_info(path)
+ assert info is not None
+ assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
+ assert info.sampling_param_cls is Wan_Animate_2_14B_SamplingParam
+
+
+def test_model_index_class_name_routes_a_checkpoint_at_any_path():
+ # A local checkpoint directory need not carry a model token in its name; its
+ # model_index.json _class_name is what identifies it.
+ with patch(
+ "sglang.multimodal_gen.registry.maybe_download_model_index",
+ return_value={"_class_name": "WanAnimate2Pipeline"},
+ ):
+ info = _get_config_info("/models/ckpt-7f3a")
+ assert info is not None
+ assert info.pipeline_config_cls is Wan_Animate_2_14B_Config
+
+
+def test_v1_animate_checkpoint_does_not_route_to_wan_animate_2():
+ # Wan2.2-Animate-14B (v1) is a different model and pipeline; a detector loosened to
+ # "animate" would pass the positive cases above and mis-route it here.
+ with patch(
+ "sglang.multimodal_gen.registry.maybe_download_model_index",
+ return_value={"_class_name": "WanAnimatePipeline"},
+ ):
+ info = _get_config_info("Wan-AI/Wan2.2-Animate-14B")
+ assert info is None or info.pipeline_config_cls is not Wan_Animate_2_14B_Config
+
+
+# /v1/videos routing: the model-specific knobs are declared extras, the reference image and
+# driving video travel in the base request fields.
+
+
+def test_video_api_routes_the_model_knobs_as_declared_extra_fields():
+ extras = get_sampling_request_extra_fields(Wan_Animate_2_14B_SamplingParam, "video")
+ assert extras == {"clip_len", "prompt_ref", "enable_audio"}
+ base_fields = {field.name for field in fields(SamplingParams)}
+ for name in extras:
+ # Not a base request field (no silent duplicate), and not on the base params
+ # either, so the declaration is what makes the API forward the value.
+ assert name not in VideoGenerationsRequest.model_fields, name
+ assert name not in base_fields, name
+ for name in ("input_reference", "reference_url", "video_path", "video_url"):
+ assert name in VideoGenerationsRequest.model_fields, name
+
+
+# Preprocessing helpers (preprocess.py): their numerics feed the conditioning tensors and have
+# no diffusers equivalent, so exact outputs are pinned.
+
+# get_frame_indices(num_frames_source, fps_source, num_frames_target, fps_target):
+# idx = clip(round(arange(num_frames_target) / fps_target * fps_source), 0, num_frames_source - 1)
+
+
+@pytest.mark.parametrize(
+ "args, expected",
+ [
+ ((100, 16, 5, 16), [0, 1, 2, 3, 4]), # fps match: identity
+ ((100, 30, 5, 15), [0, 2, 4, 6, 8]), # downsample by the fps ratio
+ ((3, 30, 6, 15), [0, 2, 2, 2, 2, 2]), # clipped to the last available frame
+ ],
+)
+def test_get_frame_indices_resamples_by_fps_ratio(args, expected):
+ assert get_frame_indices(*args) == expected
+
+
+def test_get_frame_indices_uses_banker_rounding():
+ # times*video_fps = [0.0, 0.5, 1.0, 1.5]; np.round is half-to-even, so 0.5 -> 0 and
+ # 1.5 -> 2. Python round()/int() truncation would shift the sampled frames.
+ assert get_frame_indices(10, 1, 4, 2) == [0, 0, 1, 2]
+
+
+# get_padding_len(num_frames, clip_len, num_frames_conditioning=1):
+# remaining = (num_frames - num_frames_conditioning) % (clip_len - num_frames_conditioning)
+# pad = 28 - remaining if remaining < 28 else 4 - remaining % 4
+
+_PADDING_GOLDENS = {
+ 1: 29, # remaining 0 -> pad 28
+ 28: 29, # remaining 27 -> pad 1 (boundary, still < 28)
+ 29: 33, # remaining 28 -> else branch, pad 4
+ 32: 33, # remaining 31 -> pad 1
+ 50: 53, # remaining 49 -> pad 3
+ 81: 109, # remaining 0 -> pad 28
+ 100: 109, # remaining 19 -> pad 9
+}
+
+
+def test_get_padding_len_matches_reference_goldens():
+ for input_len, expected in _PADDING_GOLDENS.items():
+ assert get_padding_len(input_len, 81) == expected, input_len
+
+
+@pytest.mark.parametrize("clip_len", [37, 81])
+def test_get_padding_len_output_is_4k_plus_1(clip_len):
+ # Padded length must land on a 4k+1 boundary (VAE temporal stride 4) for the shipped
+ # default clip length and the official one.
+ for input_len in range(1, 200):
+ assert (get_padding_len(input_len, clip_len) - 1) % 4 == 0, input_len
+
+
+# build_schedule(reference_video_frames, clip_len, num_frames_conditioning)
+# -> [_PerClipDenoisingMetadata]; frame_end_index is exclusive.
+
+
+def _frames(num_frames: int, height: int = 1, width: int = 1) -> list[np.ndarray]:
+ return [
+ np.full((height, width, 3), index % 256, np.uint8)
+ for index in range(num_frames)
+ ]
+
+
+def _as_tuples(schedule):
+ return [
+ (
+ clip.clip_index,
+ clip.frame_start_index,
+ clip.frame_end_index,
+ clip.num_frames,
+ clip.num_frames_conditioning_for_clip,
+ )
+ for clip in schedule
+ ]
+
+
+@pytest.mark.parametrize(
+ "num_frames, clip_len, num_frames_conditioning, expected",
+ [
+ # Exact tiling: 37 + 2 * (37 - 1) = 109; every clip after the first reuses one frame.
+ (109, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 73, 37, 1), (2, 72, 109, 37, 1)]),
+ # Last clip shrinks to the frames left.
+ (50, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 50, 14, 1)]),
+ # Frame 72 is already covered by clip 1, so no 1-frame clip is appended...
+ (73, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 73, 37, 1)]),
+ # ...but one extra new frame does produce a clip.
+ (74, 37, 1, [(0, 0, 37, 37, 0), (1, 36, 73, 37, 1), (2, 72, 74, 2, 1)]),
+ # Overlap follows num_frames_conditioning, not a hardcoded 1.
+ (
+ 14,
+ 5,
+ 2,
+ [(0, 0, 5, 5, 0), (1, 3, 8, 5, 2), (2, 6, 11, 5, 2), (3, 9, 14, 5, 2)],
+ ),
+ ],
+)
+def test_build_schedule_clip_boundaries(
+ num_frames, clip_len, num_frames_conditioning, expected
+):
+ schedule = build_schedule(_frames(num_frames), clip_len, num_frames_conditioning)
+ assert _as_tuples(schedule) == expected
+ covered = set()
+ for clip in schedule:
+ covered.update(range(clip.frame_start_index, clip.frame_end_index))
+ assert covered == set(range(num_frames))
+
+
+@pytest.mark.parametrize("num_frames", [5, 37, 50, 100, 236])
+def test_build_schedule_after_padding_yields_4k_plus_1_clips(num_frames):
+ # Same call chain as WanAnimate2BeforeDenoisingStage: pad first, then schedule.
+ clip_len = 37
+ padded_len = get_padding_len(num_frames, clip_len)
+ schedule = build_schedule(
+ zigzag_padding(_frames(num_frames), padded_len), clip_len, 1
+ )
+ assert schedule[-1].frame_end_index == padded_len
+ assert all(clip.num_frames % 4 == 1 for clip in schedule)
+ assert all(clip.num_frames == clip_len for clip in schedule[:-1])
+ # get_padding_len's reason to exist: the last clip generates at least 28 new frames.
+ assert schedule[-1].num_frames - 1 >= 28
+
+
+def test_build_schedule_rejects_video_too_short_for_a_clip():
+ with pytest.raises(ValueError, match="0 clips"):
+ build_schedule(_frames(1), 37, 1)
+
+
+# zigzag_padding(array, target_len): bounce the read index 0->end->0 to pad up to target_len.
+
+
+@pytest.mark.parametrize(
+ "array, target_len, expected",
+ [
+ ([0, 1, 2], 8, [0, 1, 2, 1, 0, 1, 2, 1]),
+ (["a", "b"], 5, ["a", "b", "a", "b", "a"]), # bounces at both ends every step
+ ],
+)
+def test_zigzag_padding_bounces_forward_then_backward(array, target_len, expected):
+ assert zigzag_padding(array, target_len) == expected
+
+
+def test_zigzag_padding_rejects_shortening():
+ with pytest.raises(ValueError):
+ zigzag_padding([0, 1, 2], 2)
+
+
+def test_zigzag_padding_returns_independent_deep_copies():
+ # A shallow copy would alias reused frames across the zigzag.
+ source = [[0], [1]]
+ out = zigzag_padding(source, 4)
+ out[0][0] = 99
+ assert source == [[0], [1]]
+
+
+# make_conditioning_mask(latent_t, latent_h, latent_w, num_prefix_conditioning_frames, device)
+# -> [4, latent_t, latent_h, latent_w]; the prefix is counted in pixel frames and the VAE
+# maps 4k+1 pixel frames to k+1 latent frames.
+
+
+@pytest.mark.parametrize(
+ "latent_t, prefix_pixel_frames, expected_frame_flags",
+ [
+ (1, 1, [1]), # the reference image: its single frame is given
+ (
+ 2,
+ 1,
+ [1, 0],
+ ), # one fed-back pixel frame conditions the first latent frame only
+ (3, 5, [1, 1, 0]), # 5 = 1 + 4 pixel frames -> two latent frames
+ (3, 0, [0, 0, 0]), # clip 0: nothing fed back from a previous clip
+ (
+ 3,
+ 9,
+ [1, 1, 1],
+ ), # the reference video: every one of its 9 pixel frames is given
+ ],
+)
+def test_make_conditioning_mask_prefix_in_pixel_frames_maps_to_latent_frames(
+ latent_t, prefix_pixel_frames, expected_frame_flags
+):
+ mask = make_conditioning_mask(latent_t, 1, 2, prefix_pixel_frames, device="cpu")
+ assert tuple(mask.shape) == (4, latent_t, 1, 2)
+ assert mask.dtype == torch.float32 and mask.device.type == "cpu"
+ for channel in range(4):
+ assert mask[channel, :, 0, 0].tolist() == expected_frame_flags, channel
+ # Spatially constant.
+ assert torch.equal(mask, mask[:, :, :1, :1].expand_as(mask))
+
+
+def test_make_conditioning_mask_is_channel_major():
+ # Flat golden pins the reshape/transpose exactly: [4, latent_t] channel-major.
+ mask = make_conditioning_mask(2, 1, 1, 1, device="cpu")
+ assert mask.flatten().tolist() == [1, 0, 1, 0, 1, 0, 1, 0]
+
+
+# letterbox_resize / resize_by_area / LetterboxInfo.crop: how the request ``size`` is applied
+# to the reference image and how the bars come back off every output frame.
+
+
+@pytest.mark.parametrize(
+ "source_hw, pad_axis, content_hw",
+ [((40, 80), "height", (32, 64)), ((80, 40), "width", (64, 32))],
+)
+def test_letterbox_crop_round_trips_the_content_region(source_hw, pad_axis, content_hw):
+ image = np.full((*source_hw, 3), 200, np.uint8)
+ canvas, info = letterbox_resize(image, height=64, width=64)
+ assert canvas.shape == (64, 64, 3) and canvas.dtype == np.uint8
+ assert info.pad_axis == pad_axis
+ content = info.crop(canvas[None])
+ assert content.shape == (1, *content_hw, 3)
+ # Exactly the content pixels survive the crop; the bars are the only other pixels.
+ assert (content == 200).all()
+ assert int(canvas.sum()) == int(content.sum())
+
+
+@pytest.mark.parametrize(
+ "source_hw, expected_wh",
+ [
+ ((1920, 1080), (528, 944)), # 9:16 reference under the 640x800 default budget
+ ((1080, 1920), (944, 528)),
+ ((1000, 1000), (704, 704)), # 1:1
+ ((500, 400), (640, 800)), # the budget's own 4:5 aspect fills it exactly
+ ],
+)
+def test_resize_by_area_treats_the_request_size_as_a_pixel_area_budget(
+ source_hw, expected_wh
+):
+ # The /v1/videos ``size`` and the cookbook document 640x800 -> 528x944 (9:16) and
+ # 704x704 (1:1): the request fixes the pixel area, the reference image fixes the aspect
+ # ratio, and both sides are floored to a multiple of 16.
+ image = np.zeros((*source_hw, 3), np.uint8)
+ out, _ = resize_by_area(image, 640 * 800, divisor=16)
+ assert (out.shape[1], out.shape[0]) == expected_wh
+ assert out.shape[0] * out.shape[1] <= 640 * 800
+ assert out.shape[0] % 16 == 0 and out.shape[1] % 16 == 0
+
+
+# build_clip_conditioning: per-clip tensors from the request state with stand-in encoders.
+
+
+def _request_state_for_clip_tests(frames, schedule, clip_len):
+ height, width = frames[0].shape[:2]
+ latent_h, latent_w = height // 8, width // 8
+ inputs = _WanAnimate2Inputs(
+ reference_image_path="ref.png",
+ reference_video_path="drv.mp4",
+ prompt="p",
+ prompt_ref="r",
+ negative_prompt="",
+ width=width,
+ height=height,
+ clip_len=clip_len,
+ num_frames_conditioning=1,
+ fps=16,
+ seed=0,
+ num_inference_steps=2,
+ guidance_scale=1.0,
+ enable_audio=False,
+ )
+ return WanAnimate2RequestState(
+ device=torch.device("cpu"),
+ sp_size=1,
+ generator=torch.Generator().manual_seed(0),
+ inputs=inputs,
+ letterbox_info=LetterboxInfo(pad_axis="width", padding=0, content_len=width),
+ reference_image_height=height,
+ reference_image_width=width,
+ latent_h=latent_h,
+ latent_w=latent_w,
+ reference_image_condition=torch.ones(
+ 20, 1, latent_h, latent_w, dtype=torch.bfloat16
+ ),
+ reference_image_embeddings=torch.zeros(1, 257, 1280, dtype=torch.bfloat16),
+ prompt_embeddings=torch.zeros(1, 4096, dtype=torch.bfloat16),
+ negative_prompt_embeddings=torch.zeros(1, 4096, dtype=torch.bfloat16),
+ prompt_ref_embeddings=torch.zeros(1, 4096, dtype=torch.bfloat16),
+ reference_video_frames=frames,
+ schedule=schedule,
+ num_reference_video_frames=len(frames),
+ full_clip_grid_sizes=generation_token_grid(clip_len, latent_h, latent_w),
+ )
+
+
+def test_build_clip_conditioning_shapes_and_prefix_mask_contract():
+ """Clip 0 has no fed-back frames and clip k has ``num_frames_conditioning`` of them; the
+ mask channels of ``generation_condition`` must say so frame by frame, after the
+ reference-image frame, or the DiT is told the wrong pixels are given."""
+ clip_len = 9 # 4k+1 -> 3 latent frames of its own, 4 with the reference image
+ frames = _frames(17, height=32, width=48) # two full clips sharing one frame
+ schedule = build_schedule(frames, clip_len, 1)
+ assert [clip.num_frames for clip in schedule] == [clip_len, clip_len]
+ state = _request_state_for_clip_tests(frames, schedule, clip_len)
+
+ def vae_encoder(videos):
+ return [
+ torch.zeros(16, (v.shape[1] - 1) // 4 + 1, v.shape[2] // 8, v.shape[3] // 8)
+ for v in videos
+ ]
+
+ def image_embedder(videos):
+ return torch.zeros(len(videos), 257, 1280)
+
+ clip_0 = build_clip_conditioning(
+ state, schedule[0], None, vae_encoder=vae_encoder, image_embedder=image_embedder
+ )
+ previous_frames = torch.zeros(3, 1, 32, 48)
+ clip_1 = build_clip_conditioning(
+ state,
+ schedule[1],
+ previous_frames,
+ vae_encoder=vae_encoder,
+ image_embedder=image_embedder,
+ )
+
+ for clip in (clip_0, clip_1):
+ # [20, latent_t, latent_h, latent_w]: 4 mask + 16 latent channels, latent_t = 1 + 3.
+ assert tuple(clip.generation_condition.shape) == (20, 4, 4, 6)
+ assert clip.generation_condition.dtype == torch.bfloat16
+ assert clip.generation_video_grid_sizes == (4, 2, 3)
+ assert tuple(clip.reference_video_condition.shape) == (20, 3, 4, 6)
+ assert tuple(clip.reference_video_latents.shape) == (16, 3, 4, 6)
+ assert clip.reference_video_grid_sizes == (3, 2, 3)
+ assert clip.full_clip_grid_sizes == (4, 2, 3)
+ assert tuple(clip.reference_video_frame_0_image_embeddings.shape) == (
+ 1,
+ 257,
+ 1280,
+ )
+ assert tuple(clip.init_noise.shape) == (16, 4, 4, 6)
+ # Every reference-video frame is given.
+ assert bool((clip.reference_video_condition[:4] == 1).all())
+ # Frame 0 is the reference image, given for every clip.
+ assert bool((clip.generation_condition[:4, 0] == 1).all())
+ # Clip 0: none of its own frames is given; clip 1: the one fed-back frame is.
+ assert clip_0.generation_condition[:4, 1:].abs().sum() == 0
+ assert bool((clip_1.generation_condition[:4, 1] == 1).all())
+ assert clip_1.generation_condition[:4, 2:].abs().sum() == 0
+
+ # A later clip without the previous clip's frames is a caller bug, not a silent clip 0.
+ with pytest.raises(ValueError, match="prev_clip_conditioning_frames"):
+ build_clip_conditioning(
+ state,
+ schedule[1],
+ None,
+ vae_encoder=vae_encoder,
+ image_embedder=image_embedder,
+ )
+
+
+def test_build_clip_conditioning_uses_the_stage_drawn_noise_for_clip_0_only():
+ # The before-denoising stage draws clip 0's noise into batch.latents; the clip loop must
+ # consume that draw instead of drawing again, and later clips must keep drawing.
+ clip_len = 9
+ frames = _frames(17, height=32, width=48)
+ schedule = build_schedule(frames, clip_len, 1)
+ state = _request_state_for_clip_tests(frames, schedule, clip_len)
+ stage_drawn = torch.full((16, 4, 4, 6), 7.0)
+ state.clip_0_init_noise = stage_drawn
+
+ def vae_encoder(videos):
+ return [
+ torch.zeros(16, (v.shape[1] - 1) // 4 + 1, v.shape[2] // 8, v.shape[3] // 8)
+ for v in videos
+ ]
+
+ def image_embedder(videos):
+ return torch.zeros(len(videos), 257, 1280)
+
+ clip_0 = build_clip_conditioning(
+ state, schedule[0], None, vae_encoder=vae_encoder, image_embedder=image_embedder
+ )
+ clip_1 = build_clip_conditioning(
+ state,
+ schedule[1],
+ torch.zeros(3, 1, 32, 48),
+ vae_encoder=vae_encoder,
+ image_embedder=image_embedder,
+ )
+ assert clip_0.init_noise is stage_drawn
+ assert clip_1.init_noise is not stage_drawn
+ assert tuple(clip_1.init_noise.shape) == (16, 4, 4, 6)
+ assert not torch.equal(clip_1.init_noise, stage_drawn)
+
+
+# Before-denoising stage: _WanAnimate2Inputs.from_request reads the SamplingParam-backed fields
+
+
+def test_extract_inputs_reads_request_fields_through_the_sampling_params():
+ """prompt_ref must reach the reference branch as its own text, separate from the character
+ caption that feeds the generation branch."""
+ params = Wan_Animate_2_14B_SamplingParam(
+ prompt="p",
+ image_path="ref.png",
+ video_path="drv.mp4",
+ width=640,
+ height=800,
+ clip_len=45,
+ fps=24,
+ seed=7,
+ )
+ inputs = _WanAnimate2Inputs.from_request(Req(sampling_params=params))
+
+ # Non-default values prove the fields are read off the request, not the defaults.
+ assert (inputs.clip_len, inputs.fps, inputs.seed) == (45, 24, 7)
+ assert (inputs.width, inputs.height) == (640, 800)
+ assert inputs.num_frames_conditioning == 1
+ assert (inputs.reference_image_path, inputs.reference_video_path) == (
+ "ref.png",
+ "drv.mp4",
+ )
+ assert inputs.prompt == "p"
+ assert inputs.prompt_ref == params.prompt_ref
+ assert inputs.prompt_ref != inputs.prompt
+ assert inputs.negative_prompt == params.negative_prompt
+ assert inputs.enable_audio is True
+
+
+# Carrying the audio track is a per-request knob, on by default like the official pipeline
+
+
+def test_enable_audio_is_on_by_default_and_settable_over_the_video_api():
+ assert Wan_Animate_2_14B_SamplingParam().enable_audio is True
+ assert "enable_audio" not in VideoGenerationsRequest.model_fields
+ assert (
+ "enable_audio" in Wan_Animate_2_14B_SamplingParam.video_request_extra_fields()
+ )
+ assert not hasattr(Wan_Animate_2_14B_Config(), "enable_audio")
+
+
+def _audio_stage(monkeypatch, calls: list[str], *, no_audio_track: bool = False):
+ """A before-denoising stage whose audio extractor records its calls instead of decoding
+ the file and reports either a silent reference video or a decode failure."""
+ import sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.before_denoising as mod
+
+ def fake_extract(path, *, log_info, log_warning, log_no_audio_track):
+ calls.append(path)
+ if no_audio_track:
+ log_no_audio_track(
+ "Wan-Animate-2: reference video %s has no audio track", path
+ )
+ else:
+ log_warning("Wan-Animate-2: cannot decode the audio track of %s", path)
+ return None, None
+
+ monkeypatch.setattr(mod, "extract_reference_video_audio", fake_extract)
+ stage = WanAnimate2BeforeDenoisingStage.__new__(WanAnimate2BeforeDenoisingStage)
+ stage._silent_reference_logged = False
+ stage._audio_failure_logged = False
+ stage.infos = []
+ stage.warnings = []
+ stage.debugs = []
+ stage.log_info = lambda msg, *args: stage.infos.append(msg % args)
+ stage.log_warning = lambda msg, *args: stage.warnings.append(msg % args)
+ stage.log_debug = lambda msg, *args: stage.debugs.append(msg % args)
+ return stage
+
+
+def _inputs_with_audio(enable_audio: bool):
+ params = Wan_Animate_2_14B_SamplingParam(
+ prompt="p",
+ image_path="ref.png",
+ video_path="drv.mp4",
+ width=640,
+ height=800,
+ enable_audio=enable_audio,
+ )
+ return _WanAnimate2Inputs.from_request(Req(sampling_params=params))
+
+
+def test_audio_extraction_is_skipped_when_the_request_turns_it_off(monkeypatch):
+ calls: list[str] = []
+ stage = _audio_stage(monkeypatch, calls)
+
+ assert stage._extract_audio(_inputs_with_audio(False)) == (None, None)
+ assert calls == []
+
+ stage._extract_audio(_inputs_with_audio(True))
+ assert calls == ["drv.mp4"]
+
+
+def test_silent_reference_video_logs_info_once_then_debug(monkeypatch):
+ stage = _audio_stage(monkeypatch, [], no_audio_track=True)
+ inputs = _inputs_with_audio(True)
+
+ stage._extract_audio(inputs)
+ stage._extract_audio(inputs)
+
+ assert stage.warnings == []
+ assert len(stage.infos) == 1
+ assert "no audio track" in stage.infos[0] and "enable_audio=false" in stage.infos[0]
+ assert stage.debugs == ["Wan-Animate-2: reference video drv.mp4 has no audio track"]
+
+
+def test_audio_failures_warn_once_per_process_then_go_to_debug(monkeypatch):
+ stage = _audio_stage(monkeypatch, [])
+ inputs = _inputs_with_audio(True)
+
+ stage._extract_audio(inputs)
+ stage._extract_audio(inputs)
+
+ assert len(stage.warnings) == 1
+ assert "drv.mp4" in stage.warnings[0]
+ assert "enable_audio=false" in stage.warnings[0]
+ assert "debug level" in stage.warnings[0]
+ assert stage.debugs == ["Wan-Animate-2: cannot decode the audio track of drv.mp4"]
+ # A silent reference afterwards still gets its own first-time info line.
+ assert stage.infos == []
+
+
+# extract_reference_video_audio: the shared in-process reader decodes the track at its native
+# rate; a video without an audio track is reported through the dedicated logger, any reader
+# failure through one generic warning carrying the reader's message.
+
+
+def _extract_with_fake_reader(monkeypatch, *, sample_rate, samples=None, error=None):
+ """Run the extractor with the stream probe answering ``sample_rate`` and the shared reader
+ returning ``samples`` or raising ``error``; return ``(result, logs, reader_calls)``."""
+ import sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio as mod
+
+ reader_calls: list[tuple] = []
+
+ def fake_decode(path, *, target_sr, mono):
+ reader_calls.append((path, target_sr, mono))
+ if error is not None:
+ raise error
+ return samples
+
+ monkeypatch.setattr(mod, "_audio_track_sample_rate", lambda path: sample_rate)
+ monkeypatch.setattr(mod, "decode_audio_container", fake_decode)
+ logs = {"info": [], "warning": [], "no_audio_track": []}
+ result = mod.extract_reference_video_audio(
+ "drv.mp4",
+ log_info=lambda msg, *args: logs["info"].append(msg % args),
+ log_warning=lambda msg, *args: logs["warning"].append(msg % args),
+ log_no_audio_track=lambda msg, *args: logs["no_audio_track"].append(msg % args),
+ )
+ return result, logs, reader_calls
+
+
+def test_extracted_audio_is_batched_channels_first_at_the_native_rate(monkeypatch):
+ # The reader hands back [L, C] at the requested rate.
+ samples = np.linspace(-1.0, 1.0, 2 * 480, dtype=np.float32).reshape(480, 2)
+ (audio, sample_rate), logs, reader_calls = _extract_with_fake_reader(
+ monkeypatch, sample_rate=48000, samples=samples
+ )
+ assert reader_calls == [("drv.mp4", 48000, False)]
+ assert sample_rate == 48000
+ assert audio.shape == (1, 2, 480) and audio.dtype == torch.float32
+ assert audio.is_contiguous()
+ torch.testing.assert_close(audio[0], torch.from_numpy(samples).T)
+ assert logs["warning"] == [] and logs["no_audio_track"] == []
+ assert logs["info"] == [
+ "Wan-Animate-2: extracted reference-video audio (1, 2, 480) @ 48000 Hz."
+ ]
+
+
+def test_missing_audio_track_goes_to_the_no_audio_track_logger(monkeypatch):
+ result, logs, reader_calls = _extract_with_fake_reader(
+ monkeypatch, sample_rate=None
+ )
+ assert result == (None, None)
+ assert reader_calls == []
+ assert logs["warning"] == [] and logs["info"] == []
+ assert logs["no_audio_track"] == [
+ "Wan-Animate-2: reference video drv.mp4 has no audio track; the output is silent."
+ ]
+
+
+def test_reader_failures_warn_with_the_reader_message(monkeypatch):
+ result, logs, _ = _extract_with_fake_reader(
+ monkeypatch,
+ sample_rate=44100,
+ error=ValueError("no decodable audio stream was found in the media container"),
+ )
+ assert result == (None, None)
+ assert logs["info"] == [] and logs["no_audio_track"] == []
+ assert logs["warning"] == [
+ "Wan-Animate-2: cannot decode the audio track of reference video drv.mp4 (no "
+ "decodable audio stream was found in the media container); the output is silent."
+ ]
+
+
+def test_missing_reader_dependency_warns_instead_of_raising(monkeypatch):
+ import sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.audio as mod
+
+ def probe(path):
+ raise ImportError("No module named 'av'")
+
+ monkeypatch.setattr(mod, "_audio_track_sample_rate", probe)
+ warnings: list[str] = []
+ result = mod.extract_reference_video_audio(
+ "drv.mp4",
+ log_info=lambda msg, *args: pytest.fail("no info line expected"),
+ log_warning=lambda msg, *args: warnings.append(msg % args),
+ )
+ assert result == (None, None)
+ assert warnings == [
+ "Wan-Animate-2: cannot decode the audio track of reference video drv.mp4 (No module "
+ "named 'av'); the output is silent."
+ ]
+
+
+@pytest.mark.parametrize(
+ "overrides, message",
+ [
+ ({"image_path": ["a.png", "b.png"]}, "exactly one 'image_path'"),
+ ({"video_path": []}, "exactly one 'video_path'"),
+ ({"seed": [1, 2]}, "exactly one 'seed'"),
+ ],
+)
+def test_extract_inputs_rejects_batched_single_value_fields(overrides, message):
+ # One video per request: a batched reference image, driving video or seed list must be
+ # refused here, not fail inside the stage with an index error or run the wrong seed.
+ params = Wan_Animate_2_14B_SamplingParam(
+ prompt="p", image_path="ref.png", video_path="drv.mp4"
+ )
+ batch = Req(sampling_params=params, **overrides)
+ with pytest.raises(ValueError, match=message):
+ _WanAnimate2Inputs.from_request(batch)
+
+
+# Scheduler: loaded from the checkpoint's scheduler/scheduler_config.json through the shared
+# SchedulerLoader like the other components. The stages pass the official sigma grid to
+# set_timesteps explicitly, so the loaded configuration must denoise exactly like a
+# DPMSolverMultistepScheduler built from the flow-matching settings alone.
+
+# Wan2.2-Animate-2-14B-Diffusers/scheduler/scheduler_config.json without _class_name, which
+# the loader pops before constructing the class it names.
+_CHECKPOINT_SCHEDULER_CONFIG = {
+ "_diffusers_version": "0.40.0.dev0",
+ "algorithm_type": "dpmsolver++",
+ "beta_end": 0.02,
+ "beta_schedule": "linear",
+ "beta_start": 0.0001,
+ "disable_corrector": [],
+ "dynamic_thresholding_ratio": 0.995,
+ "euler_at_final": False,
+ "final_sigmas_type": "zero",
+ "flow_shift": 5.0,
+ "lambda_min_clipped": -math.inf,
+ "lower_order_final": True,
+ "num_train_timesteps": 1000,
+ "predict_x0": True,
+ "prediction_type": "flow_prediction",
+ "rescale_betas_zero_snr": False,
+ "sample_max_value": 1.0,
+ "solver_order": 2,
+ "solver_p": None,
+ "solver_type": "midpoint",
+ "steps_offset": 0,
+ "thresholding": False,
+ "time_shift_type": "exponential",
+ "timestep_spacing": "linspace",
+ "trained_betas": None,
+ "use_beta_sigmas": False,
+ "use_dynamic_shifting": False,
+ "use_exponential_sigmas": False,
+ "use_flow_sigmas": True,
+ "use_karras_sigmas": False,
+ "use_lu_lambdas": False,
+ "variance_type": None,
+}
+
+_FLOW_MATCHING_SCHEDULER_KWARGS = {
+ "num_train_timesteps": 1000,
+ "prediction_type": "flow_prediction",
+ "use_flow_sigmas": True,
+ "flow_shift": 5.0,
+}
+
+
+def test_scheduler_is_a_loaded_component():
+ assert "scheduler" in WanAnimate2Pipeline._required_config_modules
+ assert len(WanAnimate2Pipeline._required_config_modules) == 7
+ # No pipeline-side construction: the base class hook is inherited unchanged.
+ assert "initialize_pipeline" not in vars(WanAnimate2Pipeline)
+
+
+def test_loader_keeps_the_checkpoint_scheduler_settings():
+ kwargs = _supported_init_kwargs(
+ DPMSolverMultistepScheduler, dict(_CHECKPOINT_SCHEDULER_CONFIG)
+ )
+ assert kwargs["beta_schedule"] == "linear"
+ assert kwargs["use_flow_sigmas"] is True
+ assert kwargs["prediction_type"] == "flow_prediction"
+ assert kwargs["flow_shift"] == 5.0
+ scheduler = DPMSolverMultistepScheduler(**kwargs)
+ config = scheduler._inner.config
+ assert config.algorithm_type == "dpmsolver++"
+ assert config.solver_order == 2
+ assert config.beta_schedule == "linear"
+ assert config.use_flow_sigmas is True
+ assert config.prediction_type == "flow_prediction"
+ assert config.flow_shift == 5.0
+
+
+@pytest.mark.parametrize("num_inference_steps", [4, 40])
+def test_checkpoint_scheduler_denoises_like_the_flow_matching_settings(
+ num_inference_steps,
+):
+ # beta_schedule is the only setting that differs between the two configurations
+ # (linear vs the wrapper's scaled_linear default); it only shapes the beta-derived
+ # tables, which the explicit sigma grid path never reads.
+ from_checkpoint = DPMSolverMultistepScheduler(**_CHECKPOINT_SCHEDULER_CONFIG)
+ from_settings = DPMSolverMultistepScheduler(**_FLOW_MATCHING_SCHEDULER_KWARGS)
+ sigmas = get_sampling_sigmas(num_inference_steps, 5.0)
+ from_checkpoint.set_timesteps(sigmas=sigmas)
+ from_settings.set_timesteps(sigmas=sigmas)
+ assert torch.equal(from_checkpoint.timesteps, from_settings.timesteps)
+ assert torch.equal(from_checkpoint.sigmas, from_settings.sigmas)
+ assert len(from_checkpoint.timesteps) == num_inference_steps
+
+ generator = torch.Generator().manual_seed(0)
+ sample = torch.randn(1, 16, 2, 4, 4, generator=generator)
+ model_output = torch.randn(1, 16, 2, 4, 4, generator=generator)
+ t = from_checkpoint.timesteps[0]
+ out_checkpoint = from_checkpoint.step(model_output, t, sample).prev_sample
+ out_settings = from_settings.step(model_output, t, sample).prev_sample
+ assert torch.equal(out_checkpoint, out_settings)
+
+
+# Official sampler sigma grid
+
+
+def test_official_sampler_shift_and_sigma_grid():
+ # Both literals come from the official sampler (sample_shift 5.0 and its
+ # shift * s / (1 + (shift - 1) * s) grid); the config's shift must feed that grid.
+ config = Wan_Animate_2_14B_Config()
+ assert config.flow_shift == 5.0
+ sigmas = get_sampling_sigmas(40, config.flow_shift)
+ assert len(sigmas) == 40
+ assert sigmas[0] == 1.0
+ np.testing.assert_array_equal(
+ sigmas[:3], [1.0, 0.9948979591836734, 0.9895833333333334]
+ )
+
+
+# WanAnimate2VaeAdapter wraps the sglang AutoencoderKLWan, whose encode returns the
+# DiagonalGaussianDistribution itself and whose decode returns a plain tensor (no diffusers
+# AutoencoderKLOutput / DecoderOutput wrappers).
+def test_vae_adapter_uses_the_sglang_vae_return_types():
+ class _StubWanVae(torch.nn.Module):
+ latents_mean = [0.5, -0.5]
+ latents_std = [2.0, 4.0]
+ use_parallel_decode = True
+ tiled = False
+
+ def enable_tiling(self):
+ self.tiled = True
+
+ def encode(self, x):
+ # mean = x's per-channel mean, logvar = 0, in the [B, 2C, T, H, W] layout.
+ mean = x.mean(dim=(2, 3, 4), keepdim=True).expand(-1, -1, 1, 1, 1)
+ return DiagonalGaussianDistribution(
+ torch.cat([mean, torch.zeros_like(mean)], dim=1)
+ )
+
+ def decode(self, z):
+ return z * 3.0 # a plain tensor, as the in-tree VAE returns
+
+ adapter = WanAnimate2VaeAdapter(_StubWanVae())
+
+ pixels = torch.zeros(2, 1, 4, 4)
+ pixels[0] += 2.5 # channel 0 mean 2.5, channel 1 mean 0.0
+ (latent,) = adapter.encode([pixels])
+ # (mode - mean) / std per channel: (2.5 - 0.5) / 2 = 1.0, (0.0 + 0.5) / 4 = 0.125
+ torch.testing.assert_close(latent.flatten(), torch.tensor([1.0, 0.125]))
+ assert latent.dtype == torch.float32
+
+ decoded = adapter.decode([latent])
+ # denormalize (z * std + mean) = [2.5, 0.0], then the stub's x3, clamped to [-1, 1]
+ torch.testing.assert_close(decoded.flatten(), torch.tensor([1.0, 0.0]))
+ assert decoded.shape[0] == 1 and decoded.dtype == torch.float32
+
+ args = SimpleNamespace(
+ pipeline_config=Wan_Animate_2_14B_Config(),
+ disable_autocast=False,
+ enable_torch_compile=False,
+ )
+ args.pipeline_config.vae_tiling = True
+ stage = DecodingStage(adapter)
+ with patch(
+ "sglang.multimodal_gen.runtime.pipelines_core.stages.decoding.get_local_torch_device",
+ return_value=torch.device("cpu"),
+ ):
+ raw = stage.decode_raw(latent.unsqueeze(0), args, vae_dtype=torch.float32)
+ normalized = stage.decode(latent.unsqueeze(0), args, vae_dtype=torch.float32)
+ torch.testing.assert_close(raw, decoded, rtol=0, atol=0)
+ torch.testing.assert_close(normalized, (decoded / 2 + 0.5).clamp(0, 1))
+ assert adapter.vae.tiled and adapter.use_parallel_decode
+ torch.testing.assert_close(
+ adapter.decode([latent, latent]), decoded.expand(2, -1, -1, -1, -1)
+ )
+
+
+# Deployment topology
+
+
+def test_disaggregated_roles_are_rejected_at_argument_validation():
+ """The pipeline's inter-stage state lives in batch.extra as GPU tensors, which the
+ disaggregation transport cannot carry; a split-role launch must fail before any weights
+ load instead of at the first request."""
+ for role in (RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER):
+ args = SimpleNamespace(
+ pipeline_config=Wan_Animate_2_14B_Config(), disagg_role=role
+ )
+ with pytest.raises(ValueError, match="only supports monolithic"):
+ ServerArgs._validate_disagg_capability(args)
+ ServerArgs._validate_disagg_capability(
+ SimpleNamespace(
+ pipeline_config=Wan_Animate_2_14B_Config(), disagg_role=RoleType.MONOLITHIC
+ )
+ )
+
+
+# Synthetic server warmup. The generic warmup builder supplies only ``image_path``; a
+# Wan-Animate-2 warmup request must also carry a decodable driving video that the clip
+# schedule accepts as one clip, or ``--warmup-mode server`` (forced on by
+# ``--enable-torch-compile``) can never finish. Requests are built exactly as
+# ``run_async_client_warmup`` builds them and run through the before-denoising stage with
+# stand-in encoders.
+
+_BEFORE_DENOISING = (
+ "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages"
+ ".wan_animate_2.before_denoising"
+)
+
+
+def _server_args(input_save_path: str) -> SimpleNamespace:
+ return SimpleNamespace(
+ pipeline_class_name="WanAnimate2Pipeline",
+ pipeline_config=Wan_Animate_2_14B_Config(),
+ component_precisions={},
+ enable_layerwise_nvtx_marker=False,
+ comfyui_mode=False,
+ warmup_steps=1,
+ warmup_num_frames=None,
+ warmup_resolutions=None,
+ enable_torch_compile=False,
+ enable_breakable_cuda_graph=False,
+ enable_cfg_parallel=False,
+ num_gpus=1,
+ backend="sglang",
+ # anything but "auto": keeps the auto-residency full-shape probe out of the build
+ performance_mode="balanced",
+ input_save_path=input_save_path,
+ is_arg_explicitly_set=lambda name: False,
+ )
+
+
+def _build_server_warmup_reqs(server_args: SimpleNamespace) -> list[Req]:
+ warmup_image = prepare_warmup_image_path(server_args)
+ with patch(
+ "sglang.multimodal_gen.runtime.warmup_request_builder.get_model_sampling_defaults",
+ return_value=Wan_Animate_2_14B_SamplingParam(),
+ ):
+ return build_warmup_reqs(
+ server_args,
+ warmup_resolutions=None,
+ warmup_input_path=warmup_image,
+ server_based_warmup=True,
+ )
+
+
+def _single_clip_schedule(inputs: _WanAnimate2Inputs) -> list[int]:
+ """Per-clip frame counts the stage would schedule for the request's driving video."""
+ frames = read_reference_video_frames(inputs.reference_video_path, inputs.fps)
+ assert frames.ndim == 4 and frames.shape[-1] == 3 and len(frames) > 0
+ padded = zigzag_padding(list(frames), get_padding_len(len(frames), inputs.clip_len))
+ return [
+ clip.num_frames
+ for clip in build_schedule(
+ padded, inputs.clip_len, inputs.num_frames_conditioning
+ )
+ ]
+
+
+def test_reference_video_url_uses_native_media_loading(tmp_path):
+ path = tmp_path / "reference.mp4"
+ # keep the tiny fixture's width aligned for decord's RGB frame buffer
+ write_reference_video(str(path), np.full((5, 32, 64, 3), 128, dtype=np.uint8), 16)
+ expected = read_reference_video_frames(str(path), 8)
+ assert expected.shape == (2, 32, 64, 3)
+ with patch(
+ "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages."
+ "wan_animate_2.preprocess.get_video_bytes",
+ return_value=path.read_bytes(),
+ ) as download:
+ actual = read_reference_video_frames("https://example.com/reference.mp4", 8)
+ download.assert_called_once_with("https://example.com/reference.mp4")
+ np.testing.assert_array_equal(actual, expected)
+
+
+class _StubVae:
+ """encode: [3, T, H, W] -> [16, (T-1)//4+1, H//8, W//8] fp32 zeros; the VAE module itself
+ is only handed to the residency manager."""
+
+ def __init__(self) -> None:
+ self.vae = torch.nn.Identity()
+
+ def encode(self, videos: list[torch.Tensor]) -> list[torch.Tensor]:
+ return [
+ torch.zeros(16, (v.shape[1] - 1) // 4 + 1, v.shape[2] // 8, v.shape[3] // 8)
+ for v in videos
+ ]
+
+
+class _StubImageEncoder:
+ def __init__(self) -> None:
+ self.model = torch.nn.Identity()
+
+ def visual(self, videos: list[torch.Tensor]) -> torch.Tensor:
+ return torch.zeros(len(videos), 257, 1280)
+
+
+def _before_denoising_stage(pipeline_config) -> WanAnimate2BeforeDenoisingStage:
+ stage = WanAnimate2BeforeDenoisingStage(
+ vae=_StubVae(),
+ image_encoder=_StubImageEncoder(),
+ text_encoder=torch.nn.Identity(),
+ tokenizer=object(),
+ pipeline_config=pipeline_config,
+ scheduler=DPMSolverMultistepScheduler(
+ num_train_timesteps=1000,
+ prediction_type="flow_prediction",
+ use_flow_sigmas=True,
+ flow_shift=pipeline_config.flow_shift,
+ ),
+ )
+ stage.encode_text = lambda text, *args, **kwargs: (
+ [torch.full((max(1, len(text)), 4096), 0.5)],
+ [],
+ )
+ return stage
+
+
+def test_server_warmup_request_denoises_one_clip_of_a_synthetic_driving_video(tmp_path):
+ server_args = _server_args(str(tmp_path))
+ reqs = _build_server_warmup_reqs(server_args)
+ assert len(reqs) == 1
+
+ inputs = _WanAnimate2Inputs.from_request(reqs[0])
+ assert os.path.isfile(inputs.reference_image_path)
+ assert os.path.isfile(inputs.reference_video_path)
+ assert os.path.dirname(inputs.reference_video_path) == str(tmp_path)
+ assert inputs.num_inference_steps == SERVER_WARMUP_VIDEO_STEPS
+ # Warm up at the default production clip length so the compiled attention is
+ # specialized for the geometry real requests use (the builder alone asks for 17 frames).
+ assert inputs.clip_len == Wan_Animate_2_14B_SamplingParam().clip_len
+ assert _single_clip_schedule(inputs) == [inputs.clip_len]
+ # The synthetic driving video is silent, so warmup must not decode audio for it.
+ assert inputs.enable_audio is False
+
+
+def test_warmup_requests_with_different_clip_lengths_keep_separate_driving_videos(
+ tmp_path,
+):
+ # All warmup requests are built before the first one runs, so the bounded request and
+ # the full-shape probe must not overwrite each other's video.
+ server_args = _server_args(str(tmp_path))
+ warmup_image = prepare_warmup_image_path(server_args)
+ inputs = []
+ for num_frames in (17, 45):
+ params = Wan_Animate_2_14B_SamplingParam()
+ req = Req(
+ sampling_params=params,
+ prompt="warmup",
+ image_path=[warmup_image],
+ width=640,
+ height=800,
+ num_frames=num_frames,
+ )
+ params.prepare_synthetic_warmup_request_for_queue(req, server_args)
+ inputs.append(_WanAnimate2Inputs.from_request(req))
+
+ default_len, longer = inputs
+ assert default_len.reference_video_path != longer.reference_video_path
+ assert default_len.clip_len == Wan_Animate_2_14B_SamplingParam().clip_len == 37
+ assert longer.clip_len == 45
+ assert _single_clip_schedule(default_len) == [default_len.clip_len]
+ assert _single_clip_schedule(longer) == [longer.clip_len]
+
+
+_SHARED_VIDEO_WRITER = (
+ "sglang.multimodal_gen.runtime.entrypoints.utils.post_process_sample"
+)
+
+
+def test_write_reference_video_goes_through_the_shared_video_writer(tmp_path):
+ # The synthetic warmup video is encoded by the writer request outputs go through, with
+ # the frames handed over unchanged, so it decodes wherever request outputs decode.
+ frames = np.full((9, 64, 64, 3), 200, dtype=np.uint8)
+ path = os.path.join(str(tmp_path), "warmup_driving_video_9f_30fps.mp4")
+
+ with patch(_SHARED_VIDEO_WRITER) as writer:
+ write_reference_video(path, frames, fps=30)
+
+ writer.assert_called_once()
+ (sample, data_type, fps), kwargs = writer.call_args
+ assert sample is frames
+ assert data_type == DataType.VIDEO
+ assert fps == 30
+ assert kwargs == {"save_file_path": path}
+
+
+def test_write_reference_video_names_the_shared_writer_when_it_fails(tmp_path):
+ path = os.path.join(str(tmp_path), "warmup_driving_video_9f_30fps.mp4")
+ frames = np.zeros((9, 64, 64, 3), dtype=np.uint8)
+
+ with (
+ patch(_SHARED_VIDEO_WRITER, side_effect=OSError("no video backend")),
+ pytest.raises(RuntimeError, match="shared video writer") as excinfo,
+ ):
+ write_reference_video(path, frames, fps=30)
+
+ assert path in str(excinfo.value)
+ assert "no video backend" in str(excinfo.value)
+ assert isinstance(excinfo.value.__cause__, OSError)
+
+
+@pytest.mark.parametrize("clip_len", [17, 37])
+def test_before_denoising_stage_populates_the_standard_denoising_inputs(
+ tmp_path, clip_len
+):
+ """After the before-denoising stage the Req must carry the fields the shared denoising
+ contract requires, consistent with what the clip loop will use: clip 0's initial noise
+ (the seed's first draw, which the loop consumes instead of drawing again) and the
+ official sigma grid."""
+ server_args = _server_args(str(tmp_path))
+ (req,) = _build_server_warmup_reqs(server_args)
+ req.clip_len = clip_len
+ set_global_server_args(server_args)
+ stage = _before_denoising_stage(server_args.pipeline_config)
+
+ with (
+ patch(f"{_BEFORE_DENOISING}.get_sp_world_size", return_value=1),
+ patch(
+ f"{_BEFORE_DENOISING}.get_local_torch_device",
+ return_value=torch.device("cpu"),
+ ),
+ ):
+ batch = stage(req, server_args)
+
+ request_state = request_state_from_batch(batch)
+ assert all(
+ clip.frame_start_index + clip.num_frames_conditioning_for_clip
+ < request_state.num_reference_video_frames
+ for clip in request_state.schedule
+ )
+ assert batch.generator is request_state.generator
+ assert batch.prompt_embeds == [request_state.prompt_embeddings]
+ assert batch.negative_prompt_embeds == [request_state.negative_prompt_embeddings]
+ assert batch.image_embeds == [request_state.reference_image_embeddings]
+ assert len(batch.sigmas) == batch.num_inference_steps == len(batch.timesteps)
+ assert batch.sigmas[0] == 1.0
+ assert torch.equal(
+ batch.timesteps, torch.tensor([int(s * 1000) for s in batch.sigmas])
+ )
+
+ clip_0 = build_clip_conditioning(
+ request_state,
+ request_state.schedule[0],
+ None,
+ vae_encoder=stage.vae_encoder,
+ image_embedder=stage.image_embedder,
+ )
+ assert torch.equal(batch.latents, clip_0.init_noise)
+ assert tuple(batch.raw_latent_shape) == tuple(clip_0.init_noise.shape)
+
+ # One-draw-ahead invariant: clip 0 is the seed's first draw of its shape, and the
+ # request generator now stands exactly one draw further, so the next clip gets the
+ # seed's second draw, as it did when the loop drew clip 0 itself.
+ reference = torch.Generator(device=request_state.device).manual_seed(
+ request_state.inputs.seed
+ )
+ shape = tuple(batch.latents.shape)
+ first = torch.randn(*shape, device=request_state.device, generator=reference)
+ second = torch.randn(*shape, device=request_state.device, generator=reference)
+ assert torch.equal(first, batch.latents)
+ next_clip_noise = torch.randn(
+ *shape, device=request_state.device, generator=request_state.generator
+ )
+ assert torch.equal(next_clip_noise, second)
+ assert not torch.equal(next_clip_noise, batch.latents)
+
+ denoising_verification = WanAnimate2DenoisingStage.verify_input(
+ None, batch, server_args
+ )
+ assert denoising_verification.is_valid(), (
+ denoising_verification.get_failure_summary()
+ )
+ # A request that skipped the before-denoising stage must not pass the same check.
+ bare = WanAnimate2DenoisingStage.verify_input(
+ None, Req(sampling_params=Wan_Animate_2_14B_SamplingParam()), server_args
+ )
+ assert not bare.is_valid()
diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py
new file mode 100644
index 0000000000..0bef27689d
--- /dev/null
+++ b/python/sglang/multimodal_gen/test/unit/test_wan_animate_2_parallelism.py
@@ -0,0 +1,799 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Unit tests for the Wan-Animate-2 block mask, shape math and per-clip reference K/V ownership.
+
+Single-process and CPU only, in the style of ``test_sp_shard.py`` / ``test_usp_ring_tail_pad.py``:
+``_get_block_mask_from_layout`` is called unbound with a stub ``self`` and patched module globals; the
+shape helpers are called directly. The parallelism rejections live in ``test_wan_animate_2_config.py``.
+"""
+
+from collections import OrderedDict
+from contextlib import nullcontext
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from sglang.multimodal_gen.configs.models.dits.wan_animate_2 import WanAnimate2Config
+from sglang.multimodal_gen.runtime.distributed.parallel_state import (
+ maybe_init_distributed_environment_and_model_parallel,
+ model_parallel_is_initialized,
+)
+from sglang.multimodal_gen.runtime.models.dits import wan_animate_2_block
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2 import (
+ WanAnimate2Transformer3DModel,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block import (
+ WanAnimate2TransformerBlock,
+ _apply_rope_interleaved,
+ _InContextLayout,
+ _make_score_mod,
+ _sp_scatter_out,
+)
+from sglang.multimodal_gen.runtime.models.dits.wan_animate_2_clip_conditioning import (
+ WanAnimate2ClipConditioning,
+ WanAnimate2ReferenceKV,
+)
+from sglang.multimodal_gen.runtime.models.dits.wanvideo import (
+ WanTransformer3DModel,
+ WanTransformerBlock,
+)
+from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.wan_animate_2.denoising import (
+ WanAnimate2DenoisingStage,
+)
+from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
+ ensure_distributed_env_defaults,
+)
+
+_DIT = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2"
+_BLOCK = "sglang.multimodal_gen.runtime.models.dits.wan_animate_2_block"
+_DENOISING = (
+ "sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages"
+ ".wan_animate_2.denoising"
+)
+
+
+# A short last clip against the full-length clip the mask is built for: this clip is
+# (2, 1, 2) = 2 frames x 2 tokens, the reference video (1, 1, 2), the full clip (4, 2, 2) =
+# 4 frames x 4 tokens. Every mask-side field (max_*) differs from the clip's own, so a mask
+# built from the clip's grid, or attention padding that disagrees with the mask, fails below.
+_SHORT_CLIP_LAYOUT = _InContextLayout((2, 1, 2), (1, 1, 2), (4, 2, 2))
+
+
+def _mask_probe(mask_fn):
+ def valid(q_idx, kv_idx):
+ out = mask_fn(
+ torch.tensor(0),
+ torch.tensor(0),
+ torch.tensor(q_idx),
+ torch.tensor(kv_idx),
+ )
+ return bool(out.item())
+
+ return valid
+
+
+def test_block_mask_head_agnostic_and_masks_correctly():
+ captured = {}
+
+ def fake_create_block_mask(mask_fn, *, B, H, Q_LEN, KV_LEN, device, _compile):
+ captured.update(mask_fn=mask_fn, B=B, H=H, Q_LEN=Q_LEN, KV_LEN=KV_LEN)
+ return "SENTINEL_BLOCK_MASK"
+
+ stub = SimpleNamespace(_block_masks=OrderedDict(), _device=torch.device("cpu"))
+ layout = _SHORT_CLIP_LAYOUT
+ with patch(f"{_DIT}.create_block_mask", side_effect=fake_create_block_mask) as cbm:
+ first = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout)
+ second = WanAnimate2Transformer3DModel._get_block_mask_from_layout(stub, layout)
+
+ # Cached: the second call never rebuilds.
+ assert first == "SENTINEL_BLOCK_MASK"
+ assert second is first
+ assert cbm.call_count == 1
+
+ # Head/batch-agnostic so SP head-sharding cannot change the mask.
+ assert captured["B"] is None
+ assert captured["H"] is None
+ # Sized from the full clip (16 gen + 12 reference-video tokens), each side ceil'd to 128.
+ assert captured["Q_LEN"] == 128
+ assert captured["KV_LEN"] == 256
+
+ valid = _mask_probe(captured["mask_fn"])
+ # Generation queries attend every generation token of the full clip, not the gen pad.
+ assert valid(0, 0) and valid(5, 15)
+ assert not valid(5, 16)
+ # A pad query row attends nothing.
+ assert not valid(16, 0)
+ # Frame stride is the full clip's 4 tokens per frame, not this clip's 2: query 5 is in
+ # generation frame 1 and attends reference-video frame 1 (keys 128..131) only. A mask
+ # built from the clip's own grid would put query 5 in frame 2 and query 2 in frame 1.
+ assert valid(5, 128) and valid(5, 131)
+ assert not valid(5, 132)
+ assert not valid(2, 128)
+ # Frame 0 is the reference image; it has no reference-video counterpart.
+ assert not valid(0, 128)
+ # Frame 3 attends keys 136..139; 140 is past the 12 reference-video tokens (pad).
+ assert valid(13, 136) and valid(13, 139)
+ assert not valid(13, 140)
+
+
+@pytest.mark.parametrize("cache_kind", ["rope", "mask"])
+def test_geometry_cache_bounds_and_rebuilds(cache_kind):
+ cache = OrderedDict()
+ stub = SimpleNamespace(_device=torch.device("cpu"))
+ if cache_kind == "rope":
+ limit = 4
+ stub._rope_cache = cache
+ stub.rotary_emb = SimpleNamespace(
+ forward_uncached=lambda positions: (positions.cos(), positions.sin())
+ )
+
+ def lookup(size):
+ return WanAnimate2Transformer3DModel._rope_tables(
+ stub, (0, size, 1), (0, 2), (0, 2)
+ )
+
+ context = nullcontext()
+ else:
+ limit = 2
+ stub._block_masks = cache
+
+ def lookup(size):
+ layout = _InContextLayout((size, 2, 2), (size - 1, 2, 2), (size, 2, 2))
+ return WanAnimate2Transformer3DModel._get_block_mask_from_layout(
+ stub, layout
+ )
+
+ context = patch(
+ f"{_DIT}.create_block_mask",
+ side_effect=lambda fn, **kwargs: torch.tensor(
+ [_mask_probe(fn)(0, i) for i in range(32)]
+ ),
+ )
+ with context:
+ first = lookup(2)
+ oldest_key = next(iter(cache))
+ for size in range(3, limit + 2):
+ lookup(size)
+ assert lookup(2) is first
+ lookup(limit + 2)
+ assert oldest_key in cache
+ for size in range(limit + 3, 2 * limit + 3):
+ lookup(size)
+ assert len(cache) == limit
+ assert oldest_key not in cache
+ rebuilt = lookup(2)
+ torch.testing.assert_close(rebuilt, first, rtol=0, atol=0)
+
+
+def test_attention_pads_tokens_into_the_slots_the_block_mask_admits():
+ # The mask is built once per request from the full clip while the attention pads every
+ # clip's tokens; a frame stride or reference-video offset that drifts between the two
+ # is a silent attention-pattern bug no shape check catches.
+ layout = _SHORT_CLIP_LAYOUT
+ captured = {}
+
+ def fake_flex_attention(q, k, v, *, block_mask, score_mod):
+ captured.update(q=q, k=k, v=v, block_mask=block_mask)
+ return torch.zeros_like(q)
+
+ batch, heads, dim = 1, 2, 8
+
+ def tokens(values, scale):
+ base = torch.tensor(values, dtype=torch.float32).view(1, len(values), 1, 1)
+ return (base * scale).expand(batch, len(values), heads, dim).clone()
+
+ # Token ids 1..4 are generation frame 0 (1, 2) and frame 1 (3, 4); 1, 2 x 1000 are the
+ # reference-video frame.
+ q, k, v = (
+ tokens([1, 2, 3, 4], 1.0),
+ tokens([1, 2, 3, 4], 10.0),
+ tokens([1, 2, 3, 4], 100.0),
+ )
+ reference_k, reference_v = tokens([1, 2], 1000.0), tokens([1, 2], 10000.0)
+ stub = SimpleNamespace(log_scale=0.0)
+ with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
+ out = WanAnimate2TransformerBlock._in_context_attention(
+ stub, q, k, v, reference_k, reference_v, "SENTINEL_BLOCK_MASK", layout
+ )
+
+ assert captured["block_mask"] == "SENTINEL_BLOCK_MASK"
+ assert tuple(out.shape) == (batch, 4, heads, dim)
+ q_padded = captured["q"].transpose(1, 2) # back to [B, L, N, D]
+ k_padded = captured["k"].transpose(1, 2)
+ v_padded = captured["v"].transpose(1, 2)
+ assert tuple(q_padded.shape) == (
+ batch,
+ layout.padded_generation_video_len,
+ heads,
+ dim,
+ )
+ assert tuple(k_padded.shape) == (batch, layout.padded_kv_len, heads, dim)
+
+ def slot(t, i):
+ return t[0, i, 0, 0].item()
+
+ # Generation tokens sit frame by frame at the full clip's 4-token stride.
+ assert [slot(q_padded, i) for i in range(8)] == [1, 2, 0, 0, 3, 4, 0, 0]
+ assert [slot(k_padded, i) for i in range(8)] == [10, 20, 0, 0, 30, 40, 0, 0]
+ assert q_padded[:, 8:].abs().sum() == 0
+ # Reference-video tokens start at padded_generation_video_len with the same stride.
+ reference_start = layout.padded_generation_video_len
+ assert [slot(k_padded, reference_start + i) for i in range(4)] == [1000, 2000, 0, 0]
+ assert [slot(v_padded, reference_start + i) for i in range(4)] == [
+ 10000,
+ 20000,
+ 0,
+ 0,
+ ]
+ assert k_padded[:, 8:reference_start].abs().sum() == 0
+ assert k_padded[:, reference_start + 4 :].abs().sum() == 0
+
+ # Every slot that holds a real token is one the mask built from the same layout routes:
+ # all gen pairs, and frame-1 queries (slots 4, 5) to the reference-video keys.
+ valid = _mask_probe(layout.block_mask_fn())
+ gen_slots = [0, 1, 4, 5]
+ assert all(valid(qi, ki) for qi in gen_slots for ki in gen_slots)
+ assert all(valid(qi, reference_start + ki) for qi in (4, 5) for ki in (0, 1))
+ assert not any(valid(qi, reference_start + ki) for qi in (0, 1) for ki in (0, 1))
+
+
+def test_in_context_attention_hands_flex_a_score_mod_only_when_log_scale_is_nonzero():
+ # At log_scale 0 the bias is an exact identity, so flex must run without a score_mod;
+ # otherwise it must get the cached partial (compiled flex guards on its identity).
+ layout = _SHORT_CLIP_LAYOUT
+ captured = []
+
+ def fake_flex_attention(q, k, v, *, block_mask, score_mod):
+ captured.append(score_mod)
+ return torch.zeros_like(q)
+
+ gen, reference = torch.zeros(1, 4, 2, 8), torch.zeros(1, 2, 2, 8)
+ with patch(f"{_BLOCK}._flex_attention_compiled", side_effect=fake_flex_attention):
+ for log_scale in (0.0, -1.3):
+ WanAnimate2TransformerBlock._in_context_attention(
+ SimpleNamespace(log_scale=log_scale),
+ gen,
+ gen,
+ gen,
+ reference,
+ reference,
+ "SENTINEL_BLOCK_MASK",
+ layout,
+ )
+ assert captured[0] is None
+ assert captured[1] is _make_score_mod(layout.max_tokens_per_frame, -1.3)
+
+
+def test_make_score_mod_is_cached_by_key():
+ # Compiled flex_attention guards on score_mod identity; a fresh closure per call
+ # would recompile every clip.
+ assert _make_score_mod(4, 0.0) is _make_score_mod(4, 0.0)
+ assert _make_score_mod(4, 0.0) is not _make_score_mod(4, 1.3)
+ assert _make_score_mod(4, 0.0) is not _make_score_mod(8, 0.0)
+
+
+def test_score_mod_bias_region_matches_the_official_formula():
+ # The official wan_animate_2_model.py adds log_scale to keys with index in [hw, 2*hw),
+ # hw = full-clip tokens per frame (external literal: the distilled checkpoint depends on
+ # it). In the padded [gen | reference-video] layout those are the keys of generation
+ # frame 1; the reference-video slice at padded_generation_video_len is not biased.
+ layout = _SHORT_CLIP_LAYOUT
+ hw = layout.max_tokens_per_frame # 4
+ score_mod = _make_score_mod(hw, 5.0)
+
+ def biased(kv_idx):
+ out = score_mod(torch.tensor([1.0]), 0, 0, 0, torch.tensor(kv_idx))
+ return out.item() == 6.0
+
+ probes = list(range(0, 3 * hw)) + [layout.padded_generation_video_len]
+ assert [kv for kv in probes if biased(kv)] == [4, 5, 6, 7]
+
+ # Base weights ship log_scale 0.0: the score_mod must then be an exact identity.
+ identity = _make_score_mod(hw, 0.0)
+ score = torch.randn(5)
+ for kv_idx in (0, hw, 2 * hw, layout.padded_generation_video_len):
+ assert torch.equal(
+ identity(score.clone(), 0, 0, 0, torch.tensor(kv_idx)), score
+ )
+
+
+class _FakeAllToAll4D:
+ """Single-process all_to_all_4D for SP=2: split scatter_dim into (SP, .) and fold
+ that factor onto gather_dim (and the inverse)."""
+
+ SP = 2
+
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, x, scatter_dim, gather_dim):
+ self.calls.append((scatter_dim, gather_dim))
+ b, s, h, c = x.shape
+ sp = self.SP
+ if (scatter_dim, gather_dim) == (2, 1):
+ assert h % sp == 0
+ return (
+ x.reshape(b, s, sp, h // sp, c)
+ .permute(0, 2, 1, 3, 4)
+ .reshape(b, sp * s, h // sp, c)
+ )
+ if (scatter_dim, gather_dim) == (1, 2):
+ assert s % sp == 0
+ return (
+ x.reshape(b, sp, s // sp, h, c)
+ .permute(0, 2, 1, 3, 4)
+ .reshape(b, s // sp, sp * h, c)
+ )
+ raise AssertionError((scatter_dim, gather_dim))
+
+
+def _reference_pass_stub(captured: list) -> SimpleNamespace:
+ # Minimal block for forward_ref: identity norm/modulation, q/k/v straight from the stream,
+ # attn1 records its keyword arguments.
+ def attn1(q, k, v, **kwargs):
+ captured.append({"q": q, "k": k, **kwargs})
+ return q
+
+ return SimpleNamespace(
+ local_num_heads=2,
+ _modulate=lambda t: tuple(torch.zeros(1, 1, 8) for _ in range(6)),
+ norm1=lambda x, shift, scale: x,
+ _qkv=lambda x, n: (
+ x.reshape(1, -1, n, 4),
+ x.reshape(1, -1, n, 4) + 1,
+ x.reshape(1, -1, n, 4) + 2,
+ ),
+ attn1=attn1,
+ _cross_and_ffn=lambda hidden, attn, *rest: hidden,
+ )
+
+
+def _run_reference_pass(
+ stub,
+ *,
+ seq_len: int,
+ num_valid: int,
+ sp_size: int,
+ key_mask=None,
+ k_cache=None,
+ v_cache=None,
+ cos=None,
+ sin=None,
+):
+ cos = torch.ones(num_valid, 2) if cos is None else cos
+ sin = torch.zeros(num_valid, 2) if sin is None else sin
+ with (
+ patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size),
+ patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)),
+ patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x),
+ ):
+ return WanAnimate2TransformerBlock.forward_ref(
+ stub,
+ hidden_states=torch.arange(seq_len * 8, dtype=torch.float32).reshape(
+ 1, seq_len, 8
+ ),
+ encoder_hidden_states=torch.zeros(1, 4, 8),
+ timestep_modulation=torch.zeros(1, 6, 8),
+ reference_video_rope_cos=cos,
+ reference_video_rope_sin=sin,
+ reference_video_num_tokens=num_valid,
+ index=0,
+ k_cache={} if k_cache is None else k_cache,
+ v_cache={} if v_cache is None else v_cache,
+ reference_video_key_mask=key_mask,
+ )
+
+
+@pytest.mark.parametrize("seq_len,sp_size", [(6, 1), (8, 2)])
+@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
+def test_reference_cache_reuses_rotated_key_and_preserves_padding(
+ seq_len, sp_size, dtype
+):
+ captured, keys, values = [], {}, {}
+ angles = torch.arange(12, dtype=torch.float32).view(6, 2) / 7
+ stub = _reference_pass_stub(captured)
+ qkv = stub._qkv
+ raw_keys = []
+
+ def project(x, n):
+ projected = tuple(t.to(dtype) for t in qkv(x, n))
+ raw_keys.append(projected[1])
+ return projected
+
+ stub._qkv = project
+ _run_reference_pass(
+ stub,
+ seq_len=seq_len,
+ num_valid=6,
+ sp_size=sp_size,
+ k_cache=keys,
+ v_cache=values,
+ cos=angles.cos(),
+ sin=angles.sin(),
+ )
+ raw = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 2, 4)
+ expected = torch.cat(
+ [
+ _apply_rope_interleaved(
+ (raw[:, :6] + 1).to(dtype), angles.cos(), angles.sin()
+ ),
+ (raw[:, 6:] + 1).to(dtype),
+ ],
+ dim=1,
+ )
+ torch.testing.assert_close(keys[0], expected, rtol=0, atol=0)
+ torch.testing.assert_close(values[0], (raw + 2).to(dtype), rtol=0, atol=0)
+ torch.testing.assert_close(keys[0], captured[0]["k"], rtol=0, atol=0)
+ assert keys[0].data_ptr() == raw_keys[0].data_ptr()
+
+
+def test_reference_pass_passes_no_key_mask_when_the_sequence_has_no_padding():
+ # A dense all-true key mask would route the reference stream through SDPA instead of the
+ # configured attention backend; with no padding the pass must hand attn1 attn_mask=None and
+ # RoPE the whole sequence.
+ captured: list = []
+ _run_reference_pass(
+ _reference_pass_stub(captured), seq_len=6, num_valid=6, sp_size=1
+ )
+ assert len(captured) == 1
+ assert captured[0]["attn_mask"] is None
+ assert captured[0]["q"].shape == (1, 6, 2, 4)
+
+
+@pytest.mark.parametrize("seq_len,sp_size", [(4, 1), (6, 2)])
+def test_generation_reuses_reference_cache_without_rotating_it(seq_len, sp_size):
+ stub = _reference_pass_stub([])
+ captured = []
+
+ def attend(q, k, v, ref_k, ref_v, *rest):
+ captured.append((q, k, ref_k, ref_v))
+ return q
+
+ stub._in_context_attention = attend
+ keys = {0: torch.randn(1, 2, 2, 4)}
+ values = {0: torch.randn_like(keys[0])}
+ saved_key = keys[0].clone()
+ angles = torch.arange(8, dtype=torch.float32).view(4, 2) / 7
+ hidden = torch.arange(seq_len * 8, dtype=torch.float32).view(1, seq_len, 8)
+ with (
+ patch(f"{_BLOCK}.get_sp_world_size", return_value=sp_size),
+ patch(f"{_BLOCK}._sp_gather_qkv", side_effect=lambda q, k, v: (q, k, v)),
+ patch(f"{_BLOCK}._sp_scatter_out", side_effect=lambda x: x),
+ ):
+ for _ in range(2):
+ WanAnimate2TransformerBlock.forward_gen(
+ stub,
+ hidden_states=hidden,
+ encoder_hidden_states=torch.zeros(1, 4, 8),
+ timestep_modulation=torch.zeros(1, 6, 8),
+ gen_cos=angles.cos(),
+ gen_sin=angles.sin(),
+ block_mask=None,
+ layout=_SHORT_CLIP_LAYOUT,
+ index=0,
+ k_cache=keys,
+ v_cache=values,
+ )
+ raw = hidden.view(1, seq_len, 2, 4)
+ for q, k, ref_k, ref_v in captured:
+ for actual, source in ((q, raw), (k, raw + 1)):
+ expected = torch.cat(
+ [
+ _apply_rope_interleaved(source[:, :4], angles.cos(), angles.sin()),
+ source[:, 4:],
+ ],
+ dim=1,
+ )
+ torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+ assert ref_k is keys[0] and ref_v is values[0]
+ torch.testing.assert_close(keys[0], saved_key, rtol=0, atol=0)
+
+
+def test_reference_pass_masks_the_sp_tail_padding_with_the_per_clip_mask():
+ # Gathered under SP the sequence carries tail padding: the pad keys are masked, and a mask
+ # built once per clip by the caller is used as is.
+ captured: list = []
+ _run_reference_pass(
+ _reference_pass_stub(captured), seq_len=8, num_valid=6, sp_size=2
+ )
+ mask = captured[0]["attn_mask"]
+ assert mask is not None and mask.dtype == torch.bool and tuple(mask.shape) == (1, 8)
+ assert mask[0, :6].all() and not mask[0, 6:].any()
+ assert captured[0]["skip_sequence_parallel_override"] is True
+
+ per_clip = torch.tensor([[True] * 6 + [False] * 2])
+ captured.clear()
+ _run_reference_pass(
+ _reference_pass_stub(captured),
+ seq_len=8,
+ num_valid=6,
+ sp_size=2,
+ key_mask=per_clip,
+ )
+ assert captured[0]["attn_mask"] is per_clip
+
+
+def test_gather_then_scatter_round_trips():
+ # _sp_gather_qkv: [B,S_local,N_tp,C] -> [B,S_full,N_local,C]; _sp_scatter_out inverts it.
+ fake = _FakeAllToAll4D()
+ with (
+ patch(
+ f"{_BLOCK}._sp_gather_qkv",
+ side_effect=lambda q, k, v: fake(
+ torch.cat([q, k, v]), scatter_dim=2, gather_dim=1
+ ).chunk(3),
+ ) as gather,
+ patch(
+ f"{_BLOCK}._usp_output_all_to_all",
+ side_effect=lambda x, head_dim: fake(x, scatter_dim=1, gather_dim=2),
+ ) as scatter,
+ ):
+ q = torch.arange(1 * 4 * 8 * 16, dtype=torch.float32).reshape(1, 4, 8, 16)
+ k = q + 1000.0
+ v = q + 2000.0
+
+ gq, gk, gv = wan_animate_2_block._sp_gather_qkv(q, k, v)
+ assert tuple(gq.shape) == (1, 8, 4, 16)
+ assert tuple(gk.shape) == (1, 8, 4, 16)
+ assert tuple(gv.shape) == (1, 8, 4, 16)
+
+ back = _sp_scatter_out(gq)
+ assert tuple(back.shape) == tuple(q.shape)
+ assert torch.equal(back, q)
+ gather.assert_called_once_with(q, k, v)
+ scatter.assert_called_once_with(gq, head_dim=2)
+
+ assert fake.calls == [(2, 1), (1, 2)]
+
+
+@pytest.mark.parametrize(
+ "full_clip_grid, expected",
+ [
+ # 4 frames x 4 tokens: 16 gen + 12 reference-video tokens, one 128 block each.
+ ((4, 2, 2), (4, 16, 12, 128, 128, 256)),
+ # 4 frames x 100 tokens: 400 -> 512, 300 -> 384.
+ ((4, 10, 10), (100, 400, 300, 512, 384, 896)),
+ ],
+)
+def test_layout_pads_the_full_clip_geometry_to_128_token_blocks(
+ full_clip_grid, expected
+):
+ # The reference video has no counterpart for the generation stream's reference-image
+ # frame, so its token count is one frame short of the generation stream's.
+ layout = _InContextLayout((3, 2, 2), (2, 2, 2), full_clip_grid)
+ assert (
+ layout.max_tokens_per_frame,
+ layout.max_generation_video_num_tokens,
+ layout.max_reference_video_num_tokens,
+ layout.padded_generation_video_len,
+ layout.padded_reference_video_len,
+ layout.padded_kv_len,
+ ) == expected
+
+
+class _KvTaggingTransformer:
+ """Fake DiT: ``build_reference_kv`` tags the K/V with the clip's reference-video latents,
+ ``__call__`` records the (clip_cond, reference_kv) pair each forward was handed."""
+
+ def __init__(self):
+ self.context_calls = 0
+ self.forward_calls: list[
+ tuple[WanAnimate2ClipConditioning, WanAnimate2ReferenceKV]
+ ] = []
+
+ def build_reference_kv(self, *, clip_cond):
+ latents = clip_cond.reference_video_latents
+ return WanAnimate2ReferenceKV(k={0: latents}, v={0: latents})
+
+ def prepare_context(self, prompt_embeddings, image_embeddings):
+ self.context_calls += 1
+ return prompt_embeddings
+
+ def __call__(self, *, hidden_states, clip_cond, reference_kv, **_unused):
+ self.forward_calls.append((clip_cond, reference_kv))
+ return hidden_states.unsqueeze(0)
+
+
+class _TwoStepScheduler:
+ timesteps = torch.tensor([1000.0, 500.0])
+ order = 1
+
+ def set_timesteps(self, *, sigmas, device):
+ pass
+
+ def step(self, noise_pred, t, sample, return_dict):
+ return (sample,)
+
+
+def _clip_conditioning(reference_video_latents):
+ return WanAnimate2ClipConditioning(
+ reference_video_latents=reference_video_latents,
+ generation_video_grid_sizes=(2, 1, 1),
+ reference_video_frame_0_image_embeddings=torch.zeros(1, 1, 4),
+ reference_video_condition=torch.zeros(20, 1, 2, 2),
+ prompt_ref_embeddings=torch.zeros(1, 4),
+ generation_condition=torch.zeros(20, 2, 2, 2),
+ reference_video_grid_sizes=(1, 1, 1),
+ full_clip_grid_sizes=(2, 1, 1),
+ num_frames=4,
+ init_noise=torch.zeros(16, 2, 2, 2),
+ )
+
+
+def _batch_without_perf_recording(*, is_warmup=False) -> SimpleNamespace:
+ """The clip loop reads only the perf-recording fields off the batch (and hands it to
+ set_forward_context); no metrics object means no step records."""
+ return SimpleNamespace(
+ metrics=None,
+ perf_dump_path=None,
+ is_warmup=is_warmup,
+ rollout=False,
+ return_trajectory_latents=False,
+ )
+
+
+def _fake_denoising_stage(
+ transformer, *, events: list[str]
+) -> WanAnimate2DenoisingStage:
+ """Stand-in for WanAnimate2DenoisingStage in _denoise_single_clip; the residency hooks
+ record into ``events`` so a test can read the use interval back."""
+ stage = WanAnimate2DenoisingStage.__new__(WanAnimate2DenoisingStage)
+ stage.transformer = transformer
+ stage.scheduler = _TwoStepScheduler()
+ stage.step_profile = lambda: None
+ stage._apply_nvtx_gate = lambda is_warmup: False
+ stage.log_info = lambda *args: None
+ stage.begin_declared_component_use = lambda *, component_name, module: (
+ events.append(f"begin:{component_name}")
+ )
+ stage._finish_active_component_use = lambda: events.append("finish")
+ return stage
+
+
+def _single_clip_request_state(guidance_scale: float) -> SimpleNamespace:
+ return SimpleNamespace(
+ prompt_embeddings=torch.zeros(1, 4),
+ negative_prompt_embeddings=torch.zeros(1, 4),
+ reference_image_embeddings=torch.zeros(1, 1, 4),
+ inputs=SimpleNamespace(num_inference_steps=2, guidance_scale=guidance_scale),
+ )
+
+
+@pytest.mark.parametrize("is_warmup", [False, True])
+@pytest.mark.parametrize("guidance_scale", [1.0, 3.0])
+def test_each_clip_is_denoised_with_kv_from_its_own_reference_video(
+ is_warmup, guidance_scale
+):
+ """Consecutive clips that share seed and clip index but not the reference video (two
+ requests on one DiT) must each be denoised with K/V built from their own reference video,
+ never with K/V left over from the previous one."""
+ transformer = _KvTaggingTransformer()
+ events = []
+ stage = _fake_denoising_stage(transformer, events=events)
+ stage.step_profile = lambda: events.append("profile")
+ server_args = SimpleNamespace(
+ pipeline_config=SimpleNamespace(flow_shift=5.0),
+ enable_cfg_parallel=False,
+ disable_autocast=False,
+ )
+ request_state = _single_clip_request_state(guidance_scale=guidance_scale)
+ first = _clip_conditioning(torch.full((16, 1, 2, 2), 1.0))
+ second = _clip_conditioning(torch.full((16, 1, 2, 2), 2.0))
+
+ with patch(
+ f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu")
+ ):
+ for clip_cond in (first, second):
+ WanAnimate2DenoisingStage._denoise_single_clip(
+ stage,
+ batch=_batch_without_perf_recording(is_warmup=is_warmup),
+ server_args=server_args,
+ request_state=request_state,
+ clip_condition=clip_cond,
+ )
+
+ # 2 clips x 2 steps; every forward of a clip gets the K/V built from that clip's latents.
+ branches = 2 if guidance_scale > 1 else 1
+ assert len(transformer.forward_calls) == 4 * branches
+ assert transformer.context_calls == 2 * branches
+ assert events.count("profile") == (0 if is_warmup else 4)
+ for clip_cond, reference_kv in transformer.forward_calls:
+ assert reference_kv.k[0] is clip_cond.reference_video_latents
+ assert (
+ transformer.forward_calls[0][1]
+ is not transformer.forward_calls[2 * branches][1]
+ )
+
+
+def test_dit_work_of_a_clip_runs_inside_one_declared_transformer_use():
+ """The stage declares three component uses, so the residency manager does not activate
+ the DiT at stage entry; the reference pass and every CFG forward of a clip must run after
+ the transformer use begins and before it is finished, or an offloaded DiT is never
+ brought back to the device."""
+ events: list[str] = []
+
+ class _EventTransformer(_KvTaggingTransformer):
+ def build_reference_kv(self, *, clip_cond):
+ events.append("reference_kv")
+ return super().build_reference_kv(clip_cond=clip_cond)
+
+ def __call__(self, **kwargs):
+ events.append("forward")
+ return super().__call__(**kwargs)
+
+ def prepare_context(self, *args):
+ events.append("context")
+ return super().prepare_context(*args)
+
+ stage = _fake_denoising_stage(_EventTransformer(), events=events)
+ server_args = SimpleNamespace(
+ pipeline_config=SimpleNamespace(flow_shift=5.0),
+ enable_cfg_parallel=False,
+ disable_autocast=False,
+ )
+ with patch(
+ f"{_DENOISING}.get_local_torch_device", return_value=torch.device("cpu")
+ ):
+ WanAnimate2DenoisingStage._denoise_single_clip(
+ stage,
+ batch=_batch_without_perf_recording(),
+ server_args=server_args,
+ # guidance_scale 3.0: two forwards (cond, uncond) per step.
+ request_state=_single_clip_request_state(guidance_scale=3.0),
+ clip_condition=_clip_conditioning(torch.zeros(16, 1, 2, 2)),
+ )
+
+ assert events == [
+ "begin:transformer",
+ "reference_kv",
+ "context",
+ "forward",
+ "context",
+ "forward",
+ "forward",
+ "forward",
+ "finish",
+ ]
+
+
+def _ensure_single_process_parallel_runtime() -> None:
+ # The DiT's parallel linear layers need the TP group even on a meta device.
+ if model_parallel_is_initialized():
+ return
+ ensure_distributed_env_defaults()
+ maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
+
+
+def test_dit_constructs_only_its_own_blocks():
+ """Every block the DiT constructs is the in-context block; the parent's block set is not
+ built and then replaced (40 block objects per construction, allocated on whatever device
+ the loader initializes on)."""
+ _ensure_single_process_parallel_runtime()
+ config = WanAnimate2Config()
+ config.arch_config.num_layers = 2
+ constructed: list[type] = []
+ original_init = WanTransformerBlock.__init__
+
+ def recording_init(self, *args, **kwargs):
+ constructed.append(type(self))
+ original_init(self, *args, **kwargs)
+
+ with (
+ patch.object(WanTransformerBlock, "__init__", recording_init),
+ torch.device("meta"),
+ ):
+ model = WanAnimate2Transformer3DModel(config, hf_config={})
+
+ assert [type(block) for block in model.blocks] == [WanAnimate2TransformerBlock] * 2
+ assert constructed == [WanAnimate2TransformerBlock] * 2
+
+
+def test_dit_inherits_shared_wan_projections_and_output_head():
+ assert WanAnimate2TransformerBlock._qkv is WanTransformerBlock._qkv
+ assert (
+ WanAnimate2TransformerBlock._cross_and_ffn is WanTransformerBlock._cross_and_ffn
+ )
+ assert (
+ WanAnimate2Transformer3DModel._unpatchify is WanTransformer3DModel._unpatchify
+ )
diff --git a/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py b/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py
index 7e20507065..2f70a0a364 100644
--- a/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py
+++ b/python/sglang/multimodal_gen/test/unit/test_warmup_probe_degradation.py
@@ -204,6 +204,39 @@ class TestFitAutoResidencyProbe:
assert fitted.width * fitted.height <= 832 * 480
+class TestProbeFitTerminates:
+ def test_pinned_frame_count_ends_the_shrink_ladder(self):
+ """A model whose sampling params pin num_frames must not hang the probe fit.
+
+ The lightened request comes back at the same size, so the fit must stop
+ instead of retrying the same shape forever. The request is larger than the
+ bounded warmup, so the measured-shape floor does not end the ladder first.
+ """
+ from dataclasses import dataclass
+
+ from sglang.multimodal_gen.runtime.managers.gpu_worker import (
+ fit_auto_residency_probe,
+ )
+
+ @dataclass
+ class _PinnedFrames(SamplingParams):
+ def __post_init__(self) -> None:
+ self.num_frames = 37
+ super().__post_init__()
+
+ req = Req(sampling_params=_PinnedFrames(width=576, height=1024))
+ fitted, estimate, steps = fit_auto_residency_probe(
+ req,
+ records=[_record(640, 800, 37, peak_gib=54.0)],
+ free_bytes=20 << 30,
+ total_bytes=80 << 30,
+ server_args=_server_args(),
+ )
+ assert steps == 0
+ assert (fitted.width, fitted.height, fitted.num_frames) == (576, 1024, 37)
+ assert estimate is not None and estimate > 20 << 30
+
+
class TestOutOfMemoryClassification:
def test_allocation_failures_from_libraries_count_as_out_of_memory(self):
from sglang.multimodal_gen.runtime.server_warmup import _is_out_of_memory